2024-12-11 20:10:37,190 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca
2024-12-11 20:10:37,205 main DEBUG Took 0.012759 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging
2024-12-11 20:10:37,205 main DEBUG PluginManager 'Core' found 129 plugins
2024-12-11 20:10:37,206 main DEBUG PluginManager 'Level' found 0 plugins
2024-12-11 20:10:37,207 main DEBUG PluginManager 'Lookup' found 16 plugins
2024-12-11 20:10:37,208 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-11 20:10:37,216 main DEBUG PluginManager 'TypeConverter' found 26 plugins
2024-12-11 20:10:37,230 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-11 20:10:37,232 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-11 20:10:37,233 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-11 20:10:37,233 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-11 20:10:37,234 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-11 20:10:37,234 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-11 20:10:37,235 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-11 20:10:37,236 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-11 20:10:37,236 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-11 20:10:37,237 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-11 20:10:37,238 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-11 20:10:37,238 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-11 20:10:37,239 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-11 20:10:37,239 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-11 20:10:37,240 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-11 20:10:37,240 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-11 20:10:37,241 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-11 20:10:37,241 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-11 20:10:37,242 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-11 20:10:37,242 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-11 20:10:37,243 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-11 20:10:37,243 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-11 20:10:37,244 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-11 20:10:37,244 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-11 20:10:37,245 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-11 20:10:37,245 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger].
2024-12-11 20:10:37,247 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-11 20:10:37,249 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin].
2024-12-11 20:10:37,251 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root})
2024-12-11 20:10:37,252 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout].
2024-12-11 20:10:37,253 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null")
2024-12-11 20:10:37,254 main DEBUG PluginManager 'Converter' found 47 plugins
2024-12-11 20:10:37,266 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender].
2024-12-11 20:10:37,270 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={})
2024-12-11 20:10:37,272 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR
2024-12-11 20:10:37,272 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin].
2024-12-11 20:10:37,273 main DEBUG createAppenders(={Console})
2024-12-11 20:10:37,274 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca initialized
2024-12-11 20:10:37,274 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca
2024-12-11 20:10:37,275 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca OK.
2024-12-11 20:10:37,275 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1
2024-12-11 20:10:37,276 main DEBUG OutputStream closed
2024-12-11 20:10:37,276 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true
2024-12-11 20:10:37,276 main DEBUG Appender DefaultConsole-1 stopped with status true
2024-12-11 20:10:37,277 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@6404f418 OK
2024-12-11 20:10:37,349 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6
2024-12-11 20:10:37,351 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger
2024-12-11 20:10:37,352 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector
2024-12-11 20:10:37,353 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=
2024-12-11 20:10:37,353 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory
2024-12-11 20:10:37,353 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter
2024-12-11 20:10:37,354 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper
2024-12-11 20:10:37,354 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j
2024-12-11 20:10:37,354 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl
2024-12-11 20:10:37,354 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans
2024-12-11 20:10:37,355 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase
2024-12-11 20:10:37,355 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop
2024-12-11 20:10:37,355 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers
2024-12-11 20:10:37,356 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices
2024-12-11 20:10:37,356 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig
2024-12-11 20:10:37,356 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel
2024-12-11 20:10:37,356 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore
2024-12-11 20:10:37,357 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console
2024-12-11 20:10:37,359 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-12-11 20:10:37,360 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@6dab9b6d) with optional ClassLoader: null
2024-12-11 20:10:37,360 main DEBUG Shutdown hook enabled. Registering a new one.
2024-12-11 20:10:37,361 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@6dab9b6d] started OK.
2024-12-11T20:10:37,603 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/26824f70-ac36-0745-fb30-f5180937eb59
2024-12-11 20:10:37,606 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED)
2024-12-11 20:10:37,606 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-12-11T20:10:37,616 INFO  [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestLogRolling timeout: 13 mins
2024-12-11T20:10:37,650 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=170, ProcessCount=11, AvailableMemoryMB=1296
2024-12-11T20:10:37,653 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-11T20:10:37,668 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/26824f70-ac36-0745-fb30-f5180937eb59/cluster_5e5800cd-2c49-c72c-329f-be0e65d0b9ec, deleteOnExit=true
2024-12-11T20:10:37,668 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-12-11T20:10:37,669 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/26824f70-ac36-0745-fb30-f5180937eb59/test.cache.data in system properties and HBase conf
2024-12-11T20:10:37,670 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/26824f70-ac36-0745-fb30-f5180937eb59/hadoop.tmp.dir in system properties and HBase conf
2024-12-11T20:10:37,670 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/26824f70-ac36-0745-fb30-f5180937eb59/hadoop.log.dir in system properties and HBase conf
2024-12-11T20:10:37,671 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/26824f70-ac36-0745-fb30-f5180937eb59/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-11T20:10:37,672 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/26824f70-ac36-0745-fb30-f5180937eb59/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-11T20:10:37,672 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-12-11T20:10:37,769 WARN  [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
2024-12-11T20:10:37,874 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-11T20:10:37,878 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/26824f70-ac36-0745-fb30-f5180937eb59/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-11T20:10:37,879 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/26824f70-ac36-0745-fb30-f5180937eb59/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-11T20:10:37,879 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/26824f70-ac36-0745-fb30-f5180937eb59/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-11T20:10:37,880 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/26824f70-ac36-0745-fb30-f5180937eb59/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-11T20:10:37,880 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/26824f70-ac36-0745-fb30-f5180937eb59/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-11T20:10:37,881 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/26824f70-ac36-0745-fb30-f5180937eb59/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-11T20:10:37,881 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/26824f70-ac36-0745-fb30-f5180937eb59/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-11T20:10:37,882 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/26824f70-ac36-0745-fb30-f5180937eb59/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-11T20:10:37,882 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/26824f70-ac36-0745-fb30-f5180937eb59/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-11T20:10:37,883 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/26824f70-ac36-0745-fb30-f5180937eb59/nfs.dump.dir in system properties and HBase conf
2024-12-11T20:10:37,883 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/26824f70-ac36-0745-fb30-f5180937eb59/java.io.tmpdir in system properties and HBase conf
2024-12-11T20:10:37,883 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/26824f70-ac36-0745-fb30-f5180937eb59/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-11T20:10:37,884 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/26824f70-ac36-0745-fb30-f5180937eb59/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-11T20:10:37,884 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/26824f70-ac36-0745-fb30-f5180937eb59/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-11T20:10:38,388 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-11T20:10:38,753 WARN  [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
2024-12-11T20:10:38,838 INFO  [Time-limited test {}] log.Log(170): Logging initialized @2433ms to org.eclipse.jetty.util.log.Slf4jLog
2024-12-11T20:10:38,917 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T20:10:38,988 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-11T20:10:39,008 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-11T20:10:39,009 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-11T20:10:39,010 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-11T20:10:39,023 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T20:10:39,025 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@ddc8467{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/26824f70-ac36-0745-fb30-f5180937eb59/hadoop.log.dir/,AVAILABLE}
2024-12-11T20:10:39,026 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@70be1389{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-11T20:10:39,239 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@735fa16a{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/26824f70-ac36-0745-fb30-f5180937eb59/java.io.tmpdir/jetty-localhost-45233-hadoop-hdfs-3_4_1-tests_jar-_-any-13583890554152988465/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-11T20:10:39,253 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6c26a5a3{HTTP/1.1, (http/1.1)}{localhost:45233}
2024-12-11T20:10:39,254 INFO  [Time-limited test {}] server.Server(415): Started @2849ms
2024-12-11T20:10:39,286 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-11T20:10:39,641 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T20:10:39,648 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-11T20:10:39,649 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-11T20:10:39,649 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-11T20:10:39,649 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-11T20:10:39,650 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@28778f0f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/26824f70-ac36-0745-fb30-f5180937eb59/hadoop.log.dir/,AVAILABLE}
2024-12-11T20:10:39,651 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@371e191c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-11T20:10:39,773 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7b07d1ba{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/26824f70-ac36-0745-fb30-f5180937eb59/java.io.tmpdir/jetty-localhost-37121-hadoop-hdfs-3_4_1-tests_jar-_-any-17553428015850020306/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T20:10:39,774 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@43e0a762{HTTP/1.1, (http/1.1)}{localhost:37121}
2024-12-11T20:10:39,774 INFO  [Time-limited test {}] server.Server(415): Started @3370ms
2024-12-11T20:10:39,835 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-11T20:10:40,021 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T20:10:40,047 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-11T20:10:40,057 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-11T20:10:40,058 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-11T20:10:40,058 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-11T20:10:40,059 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@11effdcd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/26824f70-ac36-0745-fb30-f5180937eb59/hadoop.log.dir/,AVAILABLE}
2024-12-11T20:10:40,060 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2d48d695{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-11T20:10:40,220 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1bf97579{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/26824f70-ac36-0745-fb30-f5180937eb59/java.io.tmpdir/jetty-localhost-38809-hadoop-hdfs-3_4_1-tests_jar-_-any-3422379013056849793/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T20:10:40,221 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@22b88bcb{HTTP/1.1, (http/1.1)}{localhost:38809}
2024-12-11T20:10:40,221 INFO  [Time-limited test {}] server.Server(415): Started @3816ms
2024-12-11T20:10:40,224 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-11T20:10:40,443 WARN  [Thread-95 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/26824f70-ac36-0745-fb30-f5180937eb59/cluster_5e5800cd-2c49-c72c-329f-be0e65d0b9ec/data/data3/current/BP-560241680-172.17.0.2-1733947838484/current, will proceed with Du for space computation calculation, 
2024-12-11T20:10:40,447 WARN  [Thread-96 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/26824f70-ac36-0745-fb30-f5180937eb59/cluster_5e5800cd-2c49-c72c-329f-be0e65d0b9ec/data/data1/current/BP-560241680-172.17.0.2-1733947838484/current, will proceed with Du for space computation calculation, 
2024-12-11T20:10:40,447 WARN  [Thread-98 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/26824f70-ac36-0745-fb30-f5180937eb59/cluster_5e5800cd-2c49-c72c-329f-be0e65d0b9ec/data/data2/current/BP-560241680-172.17.0.2-1733947838484/current, will proceed with Du for space computation calculation, 
2024-12-11T20:10:40,448 WARN  [Thread-97 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/26824f70-ac36-0745-fb30-f5180937eb59/cluster_5e5800cd-2c49-c72c-329f-be0e65d0b9ec/data/data4/current/BP-560241680-172.17.0.2-1733947838484/current, will proceed with Du for space computation calculation, 
2024-12-11T20:10:40,542 WARN  [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-11T20:10:40,543 WARN  [Thread-82 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-11T20:10:40,624 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6a43aa8dc233429a with lease ID 0x796bcd6ee0cd2179: Processing first storage report for DS-989a9c11-7ce7-48ff-86e3-293ebea318af from datanode DatanodeRegistration(127.0.0.1:32807, datanodeUuid=e1bb997a-d108-4896-aa3f-f6c561776e0d, infoPort=43923, infoSecurePort=0, ipcPort=37373, storageInfo=lv=-57;cid=testClusterID;nsid=1629681698;c=1733947838484)
2024-12-11T20:10:40,625 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6a43aa8dc233429a with lease ID 0x796bcd6ee0cd2179: from storage DS-989a9c11-7ce7-48ff-86e3-293ebea318af node DatanodeRegistration(127.0.0.1:32807, datanodeUuid=e1bb997a-d108-4896-aa3f-f6c561776e0d, infoPort=43923, infoSecurePort=0, ipcPort=37373, storageInfo=lv=-57;cid=testClusterID;nsid=1629681698;c=1733947838484), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-11T20:10:40,626 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa39d56b492d9a4a0 with lease ID 0x796bcd6ee0cd2178: Processing first storage report for DS-d9322215-295f-4a40-b488-7f9d6be8d7ad from datanode DatanodeRegistration(127.0.0.1:33959, datanodeUuid=3215035f-f1b0-4122-b293-4be79df83ad6, infoPort=44867, infoSecurePort=0, ipcPort=40749, storageInfo=lv=-57;cid=testClusterID;nsid=1629681698;c=1733947838484)
2024-12-11T20:10:40,626 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa39d56b492d9a4a0 with lease ID 0x796bcd6ee0cd2178: from storage DS-d9322215-295f-4a40-b488-7f9d6be8d7ad node DatanodeRegistration(127.0.0.1:33959, datanodeUuid=3215035f-f1b0-4122-b293-4be79df83ad6, infoPort=44867, infoSecurePort=0, ipcPort=40749, storageInfo=lv=-57;cid=testClusterID;nsid=1629681698;c=1733947838484), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T20:10:40,626 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6a43aa8dc233429a with lease ID 0x796bcd6ee0cd2179: Processing first storage report for DS-9aa15961-ba66-4d72-9191-7489e28a3a30 from datanode DatanodeRegistration(127.0.0.1:32807, datanodeUuid=e1bb997a-d108-4896-aa3f-f6c561776e0d, infoPort=43923, infoSecurePort=0, ipcPort=37373, storageInfo=lv=-57;cid=testClusterID;nsid=1629681698;c=1733947838484)
2024-12-11T20:10:40,626 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6a43aa8dc233429a with lease ID 0x796bcd6ee0cd2179: from storage DS-9aa15961-ba66-4d72-9191-7489e28a3a30 node DatanodeRegistration(127.0.0.1:32807, datanodeUuid=e1bb997a-d108-4896-aa3f-f6c561776e0d, infoPort=43923, infoSecurePort=0, ipcPort=37373, storageInfo=lv=-57;cid=testClusterID;nsid=1629681698;c=1733947838484), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-11T20:10:40,627 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa39d56b492d9a4a0 with lease ID 0x796bcd6ee0cd2178: Processing first storage report for DS-430473e8-6cea-4bae-923a-1f30fa24e811 from datanode DatanodeRegistration(127.0.0.1:33959, datanodeUuid=3215035f-f1b0-4122-b293-4be79df83ad6, infoPort=44867, infoSecurePort=0, ipcPort=40749, storageInfo=lv=-57;cid=testClusterID;nsid=1629681698;c=1733947838484)
2024-12-11T20:10:40,627 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa39d56b492d9a4a0 with lease ID 0x796bcd6ee0cd2178: from storage DS-430473e8-6cea-4bae-923a-1f30fa24e811 node DatanodeRegistration(127.0.0.1:33959, datanodeUuid=3215035f-f1b0-4122-b293-4be79df83ad6, infoPort=44867, infoSecurePort=0, ipcPort=40749, storageInfo=lv=-57;cid=testClusterID;nsid=1629681698;c=1733947838484), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T20:10:40,699 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/26824f70-ac36-0745-fb30-f5180937eb59
2024-12-11T20:10:40,786 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/26824f70-ac36-0745-fb30-f5180937eb59/cluster_5e5800cd-2c49-c72c-329f-be0e65d0b9ec/zookeeper_0, clientPort=58160, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/26824f70-ac36-0745-fb30-f5180937eb59/cluster_5e5800cd-2c49-c72c-329f-be0e65d0b9ec/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/26824f70-ac36-0745-fb30-f5180937eb59/cluster_5e5800cd-2c49-c72c-329f-be0e65d0b9ec/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-11T20:10:40,799 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=58160
2024-12-11T20:10:40,813 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T20:10:40,819 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T20:10:41,129 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33959 is added to blk_1073741825_1001 (size=7)
2024-12-11T20:10:41,130 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32807 is added to blk_1073741825_1001 (size=7)
2024-12-11T20:10:41,538 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738 with version=8
2024-12-11T20:10:41,538 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/hbase-staging
2024-12-11T20:10:41,635 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16
2024-12-11T20:10:41,893 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/9fe0640122ec:0 server-side Connection retries=45
2024-12-11T20:10:41,905 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-11T20:10:41,906 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-11T20:10:41,910 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-11T20:10:41,910 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-11T20:10:41,911 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-11T20:10:42,082 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-12-11T20:10:42,160 INFO  [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl
2024-12-11T20:10:42,169 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout
2024-12-11T20:10:42,173 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-11T20:10:42,203 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 29475 (auto-detected)
2024-12-11T20:10:42,205 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected)
2024-12-11T20:10:42,226 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35671
2024-12-11T20:10:42,249 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35671 connecting to ZooKeeper ensemble=127.0.0.1:58160
2024-12-11T20:10:42,283 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:356710x0, quorum=127.0.0.1:58160, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-11T20:10:42,286 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35671-0x100caf66b7a0000 connected
2024-12-11T20:10:42,317 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T20:10:42,319 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T20:10:42,329 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35671-0x100caf66b7a0000, quorum=127.0.0.1:58160, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-11T20:10:42,333 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738, hbase.cluster.distributed=false
2024-12-11T20:10:42,358 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35671-0x100caf66b7a0000, quorum=127.0.0.1:58160, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-11T20:10:42,364 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35671
2024-12-11T20:10:42,365 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35671
2024-12-11T20:10:42,365 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35671
2024-12-11T20:10:42,368 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35671
2024-12-11T20:10:42,369 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35671
2024-12-11T20:10:42,534 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/9fe0640122ec:0 server-side Connection retries=45
2024-12-11T20:10:42,536 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-11T20:10:42,537 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-11T20:10:42,537 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-11T20:10:42,538 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-11T20:10:42,538 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-11T20:10:42,542 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-11T20:10:42,546 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-11T20:10:42,547 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45805
2024-12-11T20:10:42,550 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45805 connecting to ZooKeeper ensemble=127.0.0.1:58160
2024-12-11T20:10:42,552 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T20:10:42,558 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T20:10:42,568 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:458050x0, quorum=127.0.0.1:58160, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-11T20:10:42,569 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:458050x0, quorum=127.0.0.1:58160, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-11T20:10:42,569 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45805-0x100caf66b7a0001 connected
2024-12-11T20:10:42,574 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-11T20:10:42,583 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-11T20:10:42,585 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45805-0x100caf66b7a0001, quorum=127.0.0.1:58160, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-11T20:10:42,591 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45805-0x100caf66b7a0001, quorum=127.0.0.1:58160, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-11T20:10:42,591 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45805
2024-12-11T20:10:42,592 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45805
2024-12-11T20:10:42,593 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45805
2024-12-11T20:10:42,595 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45805
2024-12-11T20:10:42,596 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45805
2024-12-11T20:10:42,613 DEBUG [M:0;9fe0640122ec:35671 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;9fe0640122ec:35671
2024-12-11T20:10:42,614 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/9fe0640122ec,35671,1733947841689
2024-12-11T20:10:42,621 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45805-0x100caf66b7a0001, quorum=127.0.0.1:58160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-11T20:10:42,621 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35671-0x100caf66b7a0000, quorum=127.0.0.1:58160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-11T20:10:42,624 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35671-0x100caf66b7a0000, quorum=127.0.0.1:58160, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/9fe0640122ec,35671,1733947841689
2024-12-11T20:10:42,655 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45805-0x100caf66b7a0001, quorum=127.0.0.1:58160, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-11T20:10:42,655 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35671-0x100caf66b7a0000, quorum=127.0.0.1:58160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:10:42,655 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45805-0x100caf66b7a0001, quorum=127.0.0.1:58160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:10:42,657 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35671-0x100caf66b7a0000, quorum=127.0.0.1:58160, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-11T20:10:42,658 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/9fe0640122ec,35671,1733947841689 from backup master directory
2024-12-11T20:10:42,661 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45805-0x100caf66b7a0001, quorum=127.0.0.1:58160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-11T20:10:42,661 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35671-0x100caf66b7a0000, quorum=127.0.0.1:58160, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/9fe0640122ec,35671,1733947841689
2024-12-11T20:10:42,662 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35671-0x100caf66b7a0000, quorum=127.0.0.1:58160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-11T20:10:42,662 WARN  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-11T20:10:42,663 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=9fe0640122ec,35671,1733947841689
2024-12-11T20:10:42,665 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0
2024-12-11T20:10:42,666 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0
2024-12-11T20:10:42,726 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/hbase.id] with ID: 745b692b-d827-4fdb-980f-6f35c9f82f74
2024-12-11T20:10:42,727 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/.tmp/hbase.id
2024-12-11T20:10:42,738 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32807 is added to blk_1073741826_1002 (size=42)
2024-12-11T20:10:42,738 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33959 is added to blk_1073741826_1002 (size=42)
2024-12-11T20:10:42,739 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/.tmp/hbase.id]:[hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/hbase.id]
2024-12-11T20:10:42,784 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T20:10:42,788 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-12-11T20:10:42,807 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms.
2024-12-11T20:10:42,810 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35671-0x100caf66b7a0000, quorum=127.0.0.1:58160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:10:42,810 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45805-0x100caf66b7a0001, quorum=127.0.0.1:58160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:10:42,826 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32807 is added to blk_1073741827_1003 (size=196)
2024-12-11T20:10:42,826 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33959 is added to blk_1073741827_1003 (size=196)
2024-12-11T20:10:42,842 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-11T20:10:42,844 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-11T20:10:42,850 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-11T20:10:42,881 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33959 is added to blk_1073741828_1004 (size=1189)
2024-12-11T20:10:42,882 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32807 is added to blk_1073741828_1004 (size=1189)
2024-12-11T20:10:42,900 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/MasterData/data/master/store
2024-12-11T20:10:42,920 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32807 is added to blk_1073741829_1005 (size=34)
2024-12-11T20:10:42,921 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33959 is added to blk_1073741829_1005 (size=34)
2024-12-11T20:10:42,925 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure.
2024-12-11T20:10:42,929 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T20:10:42,931 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-11T20:10:42,931 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T20:10:42,931 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T20:10:42,933 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-11T20:10:42,933 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T20:10:42,934 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T20:10:42,935 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733947842930Disabling compacts and flushes for region at 1733947842930Disabling writes for close at 1733947842933 (+3 ms)Writing region close event to WAL at 1733947842933Closed at 1733947842933
2024-12-11T20:10:42,937 WARN  [master/9fe0640122ec:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/MasterData/data/master/store/.initializing
2024-12-11T20:10:42,937 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/MasterData/WALs/9fe0640122ec,35671,1733947841689
2024-12-11T20:10:42,959 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9fe0640122ec%2C35671%2C1733947841689, suffix=, logDir=hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/MasterData/WALs/9fe0640122ec,35671,1733947841689, archiveDir=hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/MasterData/oldWALs, maxLogs=10
2024-12-11T20:10:42,967 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9fe0640122ec%2C35671%2C1733947841689.1733947842963
2024-12-11T20:10:42,989 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/MasterData/WALs/9fe0640122ec,35671,1733947841689/9fe0640122ec%2C35671%2C1733947841689.1733947842963
2024-12-11T20:10:42,999 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44867:44867),(127.0.0.1/127.0.0.1:43923:43923)]
2024-12-11T20:10:43,000 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-11T20:10:43,001 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T20:10:43,004 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T20:10:43,005 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T20:10:43,042 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-11T20:10:43,069 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-11T20:10:43,073 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:10:43,077 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T20:10:43,078 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-11T20:10:43,082 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-11T20:10:43,082 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:10:43,083 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-11T20:10:43,083 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-11T20:10:43,087 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-11T20:10:43,087 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:10:43,088 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-11T20:10:43,089 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-11T20:10:43,091 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-11T20:10:43,091 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:10:43,092 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-11T20:10:43,092 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T20:10:43,096 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-11T20:10:43,097 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-11T20:10:43,103 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T20:10:43,104 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T20:10:43,108 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-11T20:10:43,112 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T20:10:43,117 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-11T20:10:43,118 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=806989, jitterRate=0.026139840483665466}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-11T20:10:43,124 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1733947843017Initializing all the Stores at 1733947843019 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733947843020 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733947843020Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733947843021 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733947843021Cleaning up temporary data from old regions at 1733947843104 (+83 ms)Region opened successfully at 1733947843124 (+20 ms)
2024-12-11T20:10:43,125 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-11T20:10:43,161 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@332a7ae5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9fe0640122ec/172.17.0.2:0
2024-12-11T20:10:43,193 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-12-11T20:10:43,205 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-11T20:10:43,205 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-11T20:10:43,209 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-11T20:10:43,210 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec
2024-12-11T20:10:43,215 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec
2024-12-11T20:10:43,215 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-11T20:10:43,243 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-11T20:10:43,252 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35671-0x100caf66b7a0000, quorum=127.0.0.1:58160, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-11T20:10:43,257 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-12-11T20:10:43,260 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-11T20:10:43,262 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35671-0x100caf66b7a0000, quorum=127.0.0.1:58160, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-11T20:10:43,263 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-12-11T20:10:43,266 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-11T20:10:43,269 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35671-0x100caf66b7a0000, quorum=127.0.0.1:58160, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-11T20:10:43,271 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-12-11T20:10:43,272 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35671-0x100caf66b7a0000, quorum=127.0.0.1:58160, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-11T20:10:43,274 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-12-11T20:10:43,292 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35671-0x100caf66b7a0000, quorum=127.0.0.1:58160, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-11T20:10:43,293 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-11T20:10:43,297 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35671-0x100caf66b7a0000, quorum=127.0.0.1:58160, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-11T20:10:43,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35671-0x100caf66b7a0000, quorum=127.0.0.1:58160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:10:43,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45805-0x100caf66b7a0001, quorum=127.0.0.1:58160, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-11T20:10:43,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45805-0x100caf66b7a0001, quorum=127.0.0.1:58160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:10:43,300 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=9fe0640122ec,35671,1733947841689, sessionid=0x100caf66b7a0000, setting cluster-up flag (Was=false)
2024-12-11T20:10:43,313 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35671-0x100caf66b7a0000, quorum=127.0.0.1:58160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:10:43,313 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45805-0x100caf66b7a0001, quorum=127.0.0.1:58160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:10:43,318 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-11T20:10:43,320 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9fe0640122ec,35671,1733947841689
2024-12-11T20:10:43,325 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35671-0x100caf66b7a0000, quorum=127.0.0.1:58160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:10:43,325 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45805-0x100caf66b7a0001, quorum=127.0.0.1:58160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:10:43,331 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-11T20:10:43,333 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9fe0640122ec,35671,1733947841689
2024-12-11T20:10:43,340 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-12-11T20:10:43,402 INFO  [RS:0;9fe0640122ec:45805 {}] regionserver.HRegionServer(746): ClusterId : 745b692b-d827-4fdb-980f-6f35c9f82f74
2024-12-11T20:10:43,404 DEBUG [RS:0;9fe0640122ec:45805 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-11T20:10:43,409 DEBUG [RS:0;9fe0640122ec:45805 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-11T20:10:43,410 DEBUG [RS:0;9fe0640122ec:45805 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-11T20:10:43,413 DEBUG [RS:0;9fe0640122ec:45805 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-11T20:10:43,414 DEBUG [RS:0;9fe0640122ec:45805 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b2719a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9fe0640122ec/172.17.0.2:0
2024-12-11T20:10:43,414 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-12-11T20:10:43,424 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-12-11T20:10:43,431 DEBUG [RS:0;9fe0640122ec:45805 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;9fe0640122ec:45805
2024-12-11T20:10:43,433 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-11T20:10:43,435 INFO  [RS:0;9fe0640122ec:45805 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-11T20:10:43,436 INFO  [RS:0;9fe0640122ec:45805 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-11T20:10:43,436 DEBUG [RS:0;9fe0640122ec:45805 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-11T20:10:43,439 INFO  [RS:0;9fe0640122ec:45805 {}] regionserver.HRegionServer(2659): reportForDuty to master=9fe0640122ec,35671,1733947841689 with port=45805, startcode=1733947842477
2024-12-11T20:10:43,439 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 9fe0640122ec,35671,1733947841689
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-11T20:10:43,449 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/9fe0640122ec:0, corePoolSize=5, maxPoolSize=5
2024-12-11T20:10:43,449 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/9fe0640122ec:0, corePoolSize=5, maxPoolSize=5
2024-12-11T20:10:43,450 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/9fe0640122ec:0, corePoolSize=5, maxPoolSize=5
2024-12-11T20:10:43,450 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/9fe0640122ec:0, corePoolSize=5, maxPoolSize=5
2024-12-11T20:10:43,450 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/9fe0640122ec:0, corePoolSize=10, maxPoolSize=10
2024-12-11T20:10:43,450 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:10:43,451 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/9fe0640122ec:0, corePoolSize=2, maxPoolSize=2
2024-12-11T20:10:43,451 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:10:43,451 DEBUG [RS:0;9fe0640122ec:45805 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-11T20:10:43,460 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733947873460
2024-12-11T20:10:43,462 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-11T20:10:43,462 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-11T20:10:43,462 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-12-11T20:10:43,463 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-11T20:10:43,468 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-11T20:10:43,468 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-11T20:10:43,469 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-11T20:10:43,469 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-11T20:10:43,470 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-11T20:10:43,473 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:10:43,473 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-11T20:10:43,473 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-11T20:10:43,475 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-11T20:10:43,475 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-11T20:10:43,480 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-11T20:10:43,481 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-11T20:10:43,483 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/9fe0640122ec:0:becomeActiveMaster-HFileCleaner.large.0-1733947843482,5,FailOnTimeoutGroup]
2024-12-11T20:10:43,483 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/9fe0640122ec:0:becomeActiveMaster-HFileCleaner.small.0-1733947843483,5,FailOnTimeoutGroup]
2024-12-11T20:10:43,483 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-11T20:10:43,484 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-11T20:10:43,485 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-11T20:10:43,485 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-11T20:10:43,502 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33959 is added to blk_1073741831_1007 (size=1321)
2024-12-11T20:10:43,502 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32807 is added to blk_1073741831_1007 (size=1321)
2024-12-11T20:10:43,505 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-12-11T20:10:43,505 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738
2024-12-11T20:10:43,532 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32807 is added to blk_1073741832_1008 (size=32)
2024-12-11T20:10:43,533 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33959 is added to blk_1073741832_1008 (size=32)
2024-12-11T20:10:43,535 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T20:10:43,537 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-11T20:10:43,540 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-11T20:10:43,540 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:10:43,540 INFO  [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60555, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService
2024-12-11T20:10:43,541 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T20:10:43,541 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-11T20:10:43,544 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-11T20:10:43,544 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:10:43,545 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T20:10:43,546 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-11T20:10:43,547 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35671 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9fe0640122ec,45805,1733947842477
2024-12-11T20:10:43,548 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-11T20:10:43,549 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:10:43,550 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T20:10:43,550 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35671 {}] master.ServerManager(517): Registering regionserver=9fe0640122ec,45805,1733947842477
2024-12-11T20:10:43,550 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-11T20:10:43,553 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-11T20:10:43,553 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:10:43,554 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T20:10:43,554 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-11T20:10:43,555 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/hbase/meta/1588230740
2024-12-11T20:10:43,556 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/hbase/meta/1588230740
2024-12-11T20:10:43,559 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-11T20:10:43,559 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-11T20:10:43,560 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-11T20:10:43,564 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-11T20:10:43,568 DEBUG [RS:0;9fe0640122ec:45805 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738
2024-12-11T20:10:43,568 DEBUG [RS:0;9fe0640122ec:45805 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36219
2024-12-11T20:10:43,568 DEBUG [RS:0;9fe0640122ec:45805 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-11T20:10:43,571 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-11T20:10:43,572 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=734573, jitterRate=-0.06594337522983551}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-11T20:10:43,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35671-0x100caf66b7a0000, quorum=127.0.0.1:58160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-11T20:10:43,574 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1733947843535Initializing all the Stores at 1733947843536 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733947843537 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733947843537Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733947843537Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733947843537Cleaning up temporary data from old regions at 1733947843559 (+22 ms)Region opened successfully at 1733947843574 (+15 ms)
2024-12-11T20:10:43,574 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-11T20:10:43,574 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-11T20:10:43,574 DEBUG [RS:0;9fe0640122ec:45805 {}] zookeeper.ZKUtil(111): regionserver:45805-0x100caf66b7a0001, quorum=127.0.0.1:58160, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9fe0640122ec,45805,1733947842477
2024-12-11T20:10:43,575 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-11T20:10:43,575 WARN  [RS:0;9fe0640122ec:45805 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-11T20:10:43,575 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-11T20:10:43,575 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-11T20:10:43,575 INFO  [RS:0;9fe0640122ec:45805 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-11T20:10:43,575 DEBUG [RS:0;9fe0640122ec:45805 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/WALs/9fe0640122ec,45805,1733947842477
2024-12-11T20:10:43,576 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-11T20:10:43,576 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733947843574Disabling compacts and flushes for region at 1733947843574Disabling writes for close at 1733947843575 (+1 ms)Writing region close event to WAL at 1733947843576 (+1 ms)Closed at 1733947843576
2024-12-11T20:10:43,577 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9fe0640122ec,45805,1733947842477]
2024-12-11T20:10:43,581 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-11T20:10:43,581 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-12-11T20:10:43,589 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-11T20:10:43,597 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-11T20:10:43,600 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-11T20:10:43,604 INFO  [RS:0;9fe0640122ec:45805 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-11T20:10:43,617 INFO  [RS:0;9fe0640122ec:45805 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-11T20:10:43,622 INFO  [RS:0;9fe0640122ec:45805 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-11T20:10:43,622 INFO  [RS:0;9fe0640122ec:45805 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-11T20:10:43,623 INFO  [RS:0;9fe0640122ec:45805 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-11T20:10:43,628 INFO  [RS:0;9fe0640122ec:45805 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-11T20:10:43,630 INFO  [RS:0;9fe0640122ec:45805 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-11T20:10:43,630 DEBUG [RS:0;9fe0640122ec:45805 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:10:43,630 DEBUG [RS:0;9fe0640122ec:45805 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:10:43,630 DEBUG [RS:0;9fe0640122ec:45805 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:10:43,630 DEBUG [RS:0;9fe0640122ec:45805 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:10:43,630 DEBUG [RS:0;9fe0640122ec:45805 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:10:43,631 DEBUG [RS:0;9fe0640122ec:45805 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9fe0640122ec:0, corePoolSize=2, maxPoolSize=2
2024-12-11T20:10:43,631 DEBUG [RS:0;9fe0640122ec:45805 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:10:43,631 DEBUG [RS:0;9fe0640122ec:45805 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:10:43,631 DEBUG [RS:0;9fe0640122ec:45805 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:10:43,631 DEBUG [RS:0;9fe0640122ec:45805 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:10:43,631 DEBUG [RS:0;9fe0640122ec:45805 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:10:43,632 DEBUG [RS:0;9fe0640122ec:45805 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:10:43,632 DEBUG [RS:0;9fe0640122ec:45805 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9fe0640122ec:0, corePoolSize=3, maxPoolSize=3
2024-12-11T20:10:43,632 DEBUG [RS:0;9fe0640122ec:45805 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9fe0640122ec:0, corePoolSize=3, maxPoolSize=3
2024-12-11T20:10:43,633 INFO  [RS:0;9fe0640122ec:45805 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-11T20:10:43,633 INFO  [RS:0;9fe0640122ec:45805 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-11T20:10:43,633 INFO  [RS:0;9fe0640122ec:45805 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-11T20:10:43,633 INFO  [RS:0;9fe0640122ec:45805 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-11T20:10:43,633 INFO  [RS:0;9fe0640122ec:45805 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-11T20:10:43,633 INFO  [RS:0;9fe0640122ec:45805 {}] hbase.ChoreService(168): Chore ScheduledChore name=9fe0640122ec,45805,1733947842477-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-11T20:10:43,652 INFO  [RS:0;9fe0640122ec:45805 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-11T20:10:43,654 INFO  [RS:0;9fe0640122ec:45805 {}] hbase.ChoreService(168): Chore ScheduledChore name=9fe0640122ec,45805,1733947842477-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-11T20:10:43,654 INFO  [RS:0;9fe0640122ec:45805 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T20:10:43,654 INFO  [RS:0;9fe0640122ec:45805 {}] regionserver.Replication(171): 9fe0640122ec,45805,1733947842477 started
2024-12-11T20:10:43,673 INFO  [RS:0;9fe0640122ec:45805 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T20:10:43,673 INFO  [RS:0;9fe0640122ec:45805 {}] regionserver.HRegionServer(1482): Serving as 9fe0640122ec,45805,1733947842477, RpcServer on 9fe0640122ec/172.17.0.2:45805, sessionid=0x100caf66b7a0001
2024-12-11T20:10:43,674 DEBUG [RS:0;9fe0640122ec:45805 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-11T20:10:43,675 DEBUG [RS:0;9fe0640122ec:45805 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9fe0640122ec,45805,1733947842477
2024-12-11T20:10:43,675 DEBUG [RS:0;9fe0640122ec:45805 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9fe0640122ec,45805,1733947842477'
2024-12-11T20:10:43,675 DEBUG [RS:0;9fe0640122ec:45805 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-11T20:10:43,676 DEBUG [RS:0;9fe0640122ec:45805 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-11T20:10:43,677 DEBUG [RS:0;9fe0640122ec:45805 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-11T20:10:43,677 DEBUG [RS:0;9fe0640122ec:45805 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-11T20:10:43,677 DEBUG [RS:0;9fe0640122ec:45805 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9fe0640122ec,45805,1733947842477
2024-12-11T20:10:43,677 DEBUG [RS:0;9fe0640122ec:45805 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9fe0640122ec,45805,1733947842477'
2024-12-11T20:10:43,677 DEBUG [RS:0;9fe0640122ec:45805 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-11T20:10:43,678 DEBUG [RS:0;9fe0640122ec:45805 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-11T20:10:43,679 DEBUG [RS:0;9fe0640122ec:45805 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-11T20:10:43,679 INFO  [RS:0;9fe0640122ec:45805 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-11T20:10:43,679 INFO  [RS:0;9fe0640122ec:45805 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-11T20:10:43,751 WARN  [9fe0640122ec:35671 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-12-11T20:10:43,788 INFO  [RS:0;9fe0640122ec:45805 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9fe0640122ec%2C45805%2C1733947842477, suffix=, logDir=hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/WALs/9fe0640122ec,45805,1733947842477, archiveDir=hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/oldWALs, maxLogs=32
2024-12-11T20:10:43,791 INFO  [RS:0;9fe0640122ec:45805 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9fe0640122ec%2C45805%2C1733947842477.1733947843790
2024-12-11T20:10:43,799 INFO  [RS:0;9fe0640122ec:45805 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/WALs/9fe0640122ec,45805,1733947842477/9fe0640122ec%2C45805%2C1733947842477.1733947843790
2024-12-11T20:10:43,801 DEBUG [RS:0;9fe0640122ec:45805 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43923:43923),(127.0.0.1/127.0.0.1:44867:44867)]
2024-12-11T20:10:44,004 DEBUG [9fe0640122ec:35671 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-11T20:10:44,016 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=9fe0640122ec,45805,1733947842477
2024-12-11T20:10:44,023 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9fe0640122ec,45805,1733947842477, state=OPENING
2024-12-11T20:10:44,030 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-11T20:10:44,031 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35671-0x100caf66b7a0000, quorum=127.0.0.1:58160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:10:44,031 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45805-0x100caf66b7a0001, quorum=127.0.0.1:58160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:10:44,032 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-11T20:10:44,032 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-11T20:10:44,034 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-11T20:10:44,035 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=9fe0640122ec,45805,1733947842477}]
2024-12-11T20:10:44,212 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-11T20:10:44,216 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44527, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-11T20:10:44,228 INFO  [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-12-11T20:10:44,228 INFO  [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-11T20:10:44,233 INFO  [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9fe0640122ec%2C45805%2C1733947842477.meta, suffix=.meta, logDir=hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/WALs/9fe0640122ec,45805,1733947842477, archiveDir=hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/oldWALs, maxLogs=32
2024-12-11T20:10:44,235 INFO  [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 9fe0640122ec%2C45805%2C1733947842477.meta.1733947844235.meta
2024-12-11T20:10:44,245 INFO  [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/WALs/9fe0640122ec,45805,1733947842477/9fe0640122ec%2C45805%2C1733947842477.meta.1733947844235.meta
2024-12-11T20:10:44,255 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44867:44867),(127.0.0.1/127.0.0.1:43923:43923)]
2024-12-11T20:10:44,256 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-11T20:10:44,258 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-11T20:10:44,261 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-11T20:10:44,267 INFO  [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-11T20:10:44,271 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-11T20:10:44,272 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T20:10:44,272 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-12-11T20:10:44,272 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-12-11T20:10:44,275 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-11T20:10:44,277 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-11T20:10:44,277 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:10:44,278 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T20:10:44,278 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-11T20:10:44,280 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-11T20:10:44,280 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:10:44,280 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T20:10:44,281 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-11T20:10:44,282 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-11T20:10:44,282 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:10:44,283 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T20:10:44,283 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-11T20:10:44,284 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-11T20:10:44,284 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:10:44,285 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T20:10:44,285 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-11T20:10:44,287 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/hbase/meta/1588230740
2024-12-11T20:10:44,289 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/hbase/meta/1588230740
2024-12-11T20:10:44,292 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-11T20:10:44,292 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-11T20:10:44,293 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-11T20:10:44,295 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-11T20:10:44,297 INFO  [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=831855, jitterRate=0.05775958299636841}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-11T20:10:44,297 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-12-11T20:10:44,299 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1733947844272Writing region info on filesystem at 1733947844273 (+1 ms)Initializing all the Stores at 1733947844275 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733947844275Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733947844275Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733947844275Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733947844275Cleaning up temporary data from old regions at 1733947844292 (+17 ms)Running coprocessor post-open hooks at 1733947844297 (+5 ms)Region opened successfully at 1733947844299 (+2 ms)
2024-12-11T20:10:44,307 INFO  [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733947844202
2024-12-11T20:10:44,319 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-11T20:10:44,319 INFO  [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-12-11T20:10:44,321 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=9fe0640122ec,45805,1733947842477
2024-12-11T20:10:44,323 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9fe0640122ec,45805,1733947842477, state=OPEN
2024-12-11T20:10:44,328 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35671-0x100caf66b7a0000, quorum=127.0.0.1:58160, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-11T20:10:44,328 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45805-0x100caf66b7a0001, quorum=127.0.0.1:58160, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-11T20:10:44,328 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-11T20:10:44,328 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-11T20:10:44,328 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=9fe0640122ec,45805,1733947842477
2024-12-11T20:10:44,334 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-12-11T20:10:44,335 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9fe0640122ec,45805,1733947842477 in 293 msec
2024-12-11T20:10:44,342 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-12-11T20:10:44,342 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 748 msec
2024-12-11T20:10:44,343 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-11T20:10:44,343 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-12-11T20:10:44,365 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-11T20:10:44,367 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9fe0640122ec,45805,1733947842477, seqNum=-1]
2024-12-11T20:10:44,388 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-11T20:10:44,391 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43897, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-11T20:10:44,413 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0430 sec
2024-12-11T20:10:44,413 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733947844413, completionTime=-1
2024-12-11T20:10:44,416 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-11T20:10:44,416 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-12-11T20:10:44,445 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-12-11T20:10:44,446 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733947904446
2024-12-11T20:10:44,446 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733947964446
2024-12-11T20:10:44,446 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 30 msec
2024-12-11T20:10:44,449 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9fe0640122ec,35671,1733947841689-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-11T20:10:44,449 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9fe0640122ec,35671,1733947841689-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T20:10:44,449 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9fe0640122ec,35671,1733947841689-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T20:10:44,451 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-9fe0640122ec:35671, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T20:10:44,451 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-11T20:10:44,452 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-12-11T20:10:44,458 DEBUG [master/9fe0640122ec:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-12-11T20:10:44,481 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.817sec
2024-12-11T20:10:44,482 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-11T20:10:44,483 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-11T20:10:44,484 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-11T20:10:44,485 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-11T20:10:44,485 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-11T20:10:44,486 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9fe0640122ec,35671,1733947841689-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-11T20:10:44,486 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9fe0640122ec,35671,1733947841689-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-11T20:10:44,494 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-12-11T20:10:44,495 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-11T20:10:44,496 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9fe0640122ec,35671,1733947841689-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T20:10:44,515 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4731d90b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-11T20:10:44,519 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false
2024-12-11T20:10:44,519 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512
2024-12-11T20:10:44,524 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9fe0640122ec,35671,-1 for getting cluster id
2024-12-11T20:10:44,528 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-12-11T20:10:44,537 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '745b692b-d827-4fdb-980f-6f35c9f82f74'
2024-12-11T20:10:44,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-12-11T20:10:44,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "745b692b-d827-4fdb-980f-6f35c9f82f74"

2024-12-11T20:10:44,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e9c12a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-11T20:10:44,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9fe0640122ec,35671,-1]
2024-12-11T20:10:44,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-12-11T20:10:44,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T20:10:44,549 INFO  [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46308, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-12-11T20:10:44,552 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2eb3700d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-11T20:10:44,553 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-11T20:10:44,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9fe0640122ec,45805,1733947842477, seqNum=-1]
2024-12-11T20:10:44,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-11T20:10:44,564 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34178, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-11T20:10:44,593 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=9fe0640122ec,35671,1733947841689
2024-12-11T20:10:44,593 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T20:10:44,604 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-12-11T20:10:44,608 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-12-11T20:10:44,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 9fe0640122ec,35671,1733947841689
2024-12-11T20:10:44,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3c0d2759
2024-12-11T20:10:44,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-12-11T20:10:44,622 INFO  [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46324, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-12-11T20:10:44,624 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35671 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-12-11T20:10:44,624 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35671 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-12-11T20:10:44,628 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35671 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testSlowSyncLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-11T20:10:44,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35671 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling
2024-12-11T20:10:44,641 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-12-11T20:10:44,643 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35671 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testSlowSyncLogRolling"
 procId is: 4
2024-12-11T20:10:44,644 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:10:44,647 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-11T20:10:44,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-11T20:10:44,705 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33959 is added to blk_1073741835_1011 (size=389)
2024-12-11T20:10:44,706 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32807 is added to blk_1073741835_1011 (size=389)
2024-12-11T20:10:44,710 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5406b284938f2b29683221552ae824d8, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1733947844623.5406b284938f2b29683221552ae824d8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testSlowSyncLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738
2024-12-11T20:10:44,720 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33959 is added to blk_1073741836_1012 (size=72)
2024-12-11T20:10:44,721 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32807 is added to blk_1073741836_1012 (size=72)
2024-12-11T20:10:44,723 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testSlowSyncLogRolling,,1733947844623.5406b284938f2b29683221552ae824d8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T20:10:44,723 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1722): Closing 5406b284938f2b29683221552ae824d8, disabling compactions & flushes
2024-12-11T20:10:44,723 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testSlowSyncLogRolling,,1733947844623.5406b284938f2b29683221552ae824d8.
2024-12-11T20:10:44,723 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testSlowSyncLogRolling,,1733947844623.5406b284938f2b29683221552ae824d8.
2024-12-11T20:10:44,723 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1733947844623.5406b284938f2b29683221552ae824d8. after waiting 0 ms
2024-12-11T20:10:44,723 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1733947844623.5406b284938f2b29683221552ae824d8.
2024-12-11T20:10:44,723 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testSlowSyncLogRolling,,1733947844623.5406b284938f2b29683221552ae824d8.
2024-12-11T20:10:44,724 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5406b284938f2b29683221552ae824d8:
Waiting for close lock at 1733947844723Disabling compacts and flushes for region at 1733947844723Disabling writes for close at 1733947844723Writing region close event to WAL at 1733947844723Closed at 1733947844723
2024-12-11T20:10:44,726 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ADD_TO_META
2024-12-11T20:10:44,732 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testSlowSyncLogRolling,,1733947844623.5406b284938f2b29683221552ae824d8.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1733947844726"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733947844726"}]},"ts":"1733947844726"}
2024-12-11T20:10:44,738 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-12-11T20:10:44,740 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-11T20:10:44,743 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733947844740"}]},"ts":"1733947844740"}
2024-12-11T20:10:44,749 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLING in hbase:meta
2024-12-11T20:10:44,751 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=5406b284938f2b29683221552ae824d8, ASSIGN}]
2024-12-11T20:10:44,753 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=5406b284938f2b29683221552ae824d8, ASSIGN
2024-12-11T20:10:44,755 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=5406b284938f2b29683221552ae824d8, ASSIGN; state=OFFLINE, location=9fe0640122ec,45805,1733947842477; forceNewPlan=false, retain=false
2024-12-11T20:10:44,907 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=5406b284938f2b29683221552ae824d8, regionState=OPENING, regionLocation=9fe0640122ec,45805,1733947842477
2024-12-11T20:10:44,912 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=5406b284938f2b29683221552ae824d8, ASSIGN because future has completed
2024-12-11T20:10:44,913 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5406b284938f2b29683221552ae824d8, server=9fe0640122ec,45805,1733947842477}]
2024-12-11T20:10:45,075 INFO  [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testSlowSyncLogRolling,,1733947844623.5406b284938f2b29683221552ae824d8.
2024-12-11T20:10:45,075 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 5406b284938f2b29683221552ae824d8, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1733947844623.5406b284938f2b29683221552ae824d8.', STARTKEY => '', ENDKEY => ''}
2024-12-11T20:10:45,076 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testSlowSyncLogRolling 5406b284938f2b29683221552ae824d8
2024-12-11T20:10:45,076 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testSlowSyncLogRolling,,1733947844623.5406b284938f2b29683221552ae824d8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T20:10:45,076 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 5406b284938f2b29683221552ae824d8
2024-12-11T20:10:45,076 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 5406b284938f2b29683221552ae824d8
2024-12-11T20:10:45,079 INFO  [StoreOpener-5406b284938f2b29683221552ae824d8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 5406b284938f2b29683221552ae824d8 
2024-12-11T20:10:45,081 INFO  [StoreOpener-5406b284938f2b29683221552ae824d8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5406b284938f2b29683221552ae824d8 columnFamilyName info
2024-12-11T20:10:45,081 DEBUG [StoreOpener-5406b284938f2b29683221552ae824d8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:10:45,082 INFO  [StoreOpener-5406b284938f2b29683221552ae824d8-1 {}] regionserver.HStore(327): Store=5406b284938f2b29683221552ae824d8/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-11T20:10:45,083 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 5406b284938f2b29683221552ae824d8
2024-12-11T20:10:45,084 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/default/TestLogRolling-testSlowSyncLogRolling/5406b284938f2b29683221552ae824d8
2024-12-11T20:10:45,085 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/default/TestLogRolling-testSlowSyncLogRolling/5406b284938f2b29683221552ae824d8
2024-12-11T20:10:45,086 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 5406b284938f2b29683221552ae824d8
2024-12-11T20:10:45,086 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 5406b284938f2b29683221552ae824d8
2024-12-11T20:10:45,088 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 5406b284938f2b29683221552ae824d8
2024-12-11T20:10:45,092 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/default/TestLogRolling-testSlowSyncLogRolling/5406b284938f2b29683221552ae824d8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-11T20:10:45,093 INFO  [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 5406b284938f2b29683221552ae824d8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=749181, jitterRate=-0.04736803472042084}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-11T20:10:45,093 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5406b284938f2b29683221552ae824d8
2024-12-11T20:10:45,095 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 5406b284938f2b29683221552ae824d8:
Running coprocessor pre-open hook at 1733947845076Writing region info on filesystem at 1733947845076Initializing all the Stores at 1733947845078 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733947845078Cleaning up temporary data from old regions at 1733947845086 (+8 ms)Running coprocessor post-open hooks at 1733947845093 (+7 ms)Region opened successfully at 1733947845095 (+2 ms)
2024-12-11T20:10:45,097 INFO  [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testSlowSyncLogRolling,,1733947844623.5406b284938f2b29683221552ae824d8., pid=6, masterSystemTime=1733947845068
2024-12-11T20:10:45,120 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testSlowSyncLogRolling,,1733947844623.5406b284938f2b29683221552ae824d8.
2024-12-11T20:10:45,120 INFO  [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testSlowSyncLogRolling,,1733947844623.5406b284938f2b29683221552ae824d8.
2024-12-11T20:10:45,126 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=5406b284938f2b29683221552ae824d8, regionState=OPEN, openSeqNum=2, regionLocation=9fe0640122ec,45805,1733947842477
2024-12-11T20:10:45,132 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5406b284938f2b29683221552ae824d8, server=9fe0640122ec,45805,1733947842477 because future has completed
2024-12-11T20:10:45,139 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-12-11T20:10:45,139 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 5406b284938f2b29683221552ae824d8, server=9fe0640122ec,45805,1733947842477 in 221 msec
2024-12-11T20:10:45,143 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-12-11T20:10:45,143 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=5406b284938f2b29683221552ae824d8, ASSIGN in 388 msec
2024-12-11T20:10:45,144 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-11T20:10:45,145 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733947845145"}]},"ts":"1733947845145"}
2024-12-11T20:10:45,148 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLED in hbase:meta
2024-12-11T20:10:45,154 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_POST_OPERATION
2024-12-11T20:10:45,157 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling in 524 msec
2024-12-11T20:10:49,775 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties
2024-12-11T20:10:49,819 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-12-11T20:10:49,820 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testSlowSyncLogRolling'
2024-12-11T20:10:52,156 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-12-11T20:10:52,157 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-12-11T20:10:52,158 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling
2024-12-11T20:10:52,158 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling Metrics about Tables on a single HBase RegionServer
2024-12-11T20:10:52,159 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-11T20:10:52,160 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-12-11T20:10:52,160 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-12-11T20:10:52,160 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-12-11T20:10:54,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35671 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-11T20:10:54,739 INFO  [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testSlowSyncLogRolling completed
2024-12-11T20:10:54,741 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testSlowSyncLogRolling,, stopping at row=TestLogRolling-testSlowSyncLogRolling ,, for max=2147483647 with caching=100
2024-12-11T20:10:54,748 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testSlowSyncLogRolling
2024-12-11T20:10:54,749 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testSlowSyncLogRolling,,1733947844623.5406b284938f2b29683221552ae824d8.
2024-12-11T20:10:54,750 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9fe0640122ec%2C45805%2C1733947842477.1733947854749
2024-12-11T20:10:54,759 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:10:54,760 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:10:54,760 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:10:54,760 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:10:54,760 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:10:54,761 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/WALs/9fe0640122ec,45805,1733947842477/9fe0640122ec%2C45805%2C1733947842477.1733947843790 with entries=1, filesize=443 B; new WAL /user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/WALs/9fe0640122ec,45805,1733947842477/9fe0640122ec%2C45805%2C1733947842477.1733947854749
2024-12-11T20:10:54,762 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43923:43923),(127.0.0.1/127.0.0.1:44867:44867)]
2024-12-11T20:10:54,762 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/WALs/9fe0640122ec,45805,1733947842477/9fe0640122ec%2C45805%2C1733947842477.1733947843790 is not closed yet, will try archiving it next time
2024-12-11T20:10:54,764 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33959 is added to blk_1073741833_1009 (size=451)
2024-12-11T20:10:54,764 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32807 is added to blk_1073741833_1009 (size=451)
2024-12-11T20:10:54,770 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/WALs/9fe0640122ec,45805,1733947842477/9fe0640122ec%2C45805%2C1733947842477.1733947843790 to hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/oldWALs/9fe0640122ec%2C45805%2C1733947842477.1733947843790
2024-12-11T20:10:54,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testSlowSyncLogRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testSlowSyncLogRolling,,1733947844623.5406b284938f2b29683221552ae824d8., hostname=9fe0640122ec,45805,1733947842477, seqNum=2]
2024-12-11T20:11:06,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45805 {}] regionserver.HRegion(8855): Flush requested on 5406b284938f2b29683221552ae824d8
2024-12-11T20:11:06,809 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5406b284938f2b29683221552ae824d8 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-11T20:11:06,892 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/default/TestLogRolling-testSlowSyncLogRolling/5406b284938f2b29683221552ae824d8/.tmp/info/e5656d56806a4366a53b430e3a62224a is 1080, key is row0001/info:/1733947854773/Put/seqid=0
2024-12-11T20:11:06,905 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32807 is added to blk_1073741838_1014 (size=12509)
2024-12-11T20:11:06,906 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33959 is added to blk_1073741838_1014 (size=12509)
2024-12-11T20:11:06,907 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/default/TestLogRolling-testSlowSyncLogRolling/5406b284938f2b29683221552ae824d8/.tmp/info/e5656d56806a4366a53b430e3a62224a
2024-12-11T20:11:06,956 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/default/TestLogRolling-testSlowSyncLogRolling/5406b284938f2b29683221552ae824d8/.tmp/info/e5656d56806a4366a53b430e3a62224a as hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/default/TestLogRolling-testSlowSyncLogRolling/5406b284938f2b29683221552ae824d8/info/e5656d56806a4366a53b430e3a62224a
2024-12-11T20:11:06,968 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/default/TestLogRolling-testSlowSyncLogRolling/5406b284938f2b29683221552ae824d8/info/e5656d56806a4366a53b430e3a62224a, entries=7, sequenceid=11, filesize=12.2 K
2024-12-11T20:11:06,975 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 5406b284938f2b29683221552ae824d8 in 167ms, sequenceid=11, compaction requested=false
2024-12-11T20:11:06,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5406b284938f2b29683221552ae824d8:

2024-12-11T20:11:10,696 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-11T20:11:14,827 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9fe0640122ec%2C45805%2C1733947842477.1733947874827
2024-12-11T20:11:15,042 INFO  [Time-limited test {}] wal.AbstractFSWAL(1368): Slow sync cost: 212 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:32807,DS-989a9c11-7ce7-48ff-86e3-293ebea318af,DISK], DatanodeInfoWithStorage[127.0.0.1:33959,DS-d9322215-295f-4a40-b488-7f9d6be8d7ad,DISK]]
2024-12-11T20:11:15,042 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:11:15,042 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:11:15,043 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:11:15,043 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:11:15,043 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:11:15,043 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/WALs/9fe0640122ec,45805,1733947842477/9fe0640122ec%2C45805%2C1733947842477.1733947854749 with entries=12, filesize=12.10 KB; new WAL /user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/WALs/9fe0640122ec,45805,1733947842477/9fe0640122ec%2C45805%2C1733947842477.1733947874827
2024-12-11T20:11:15,045 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43923:43923),(127.0.0.1/127.0.0.1:44867:44867)]
2024-12-11T20:11:15,045 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/WALs/9fe0640122ec,45805,1733947842477/9fe0640122ec%2C45805%2C1733947842477.1733947854749 is not closed yet, will try archiving it next time
2024-12-11T20:11:15,048 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33959 is added to blk_1073741837_1013 (size=12399)
2024-12-11T20:11:15,049 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32807 is added to blk_1073741837_1013 (size=12399)
2024-12-11T20:11:15,249 INFO  [FSHLog-0-hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738-prefix:9fe0640122ec,45805,1733947842477 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:32807,DS-989a9c11-7ce7-48ff-86e3-293ebea318af,DISK], DatanodeInfoWithStorage[127.0.0.1:33959,DS-d9322215-295f-4a40-b488-7f9d6be8d7ad,DISK]]
2024-12-11T20:11:17,454 INFO  [FSHLog-0-hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738-prefix:9fe0640122ec,45805,1733947842477 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:32807,DS-989a9c11-7ce7-48ff-86e3-293ebea318af,DISK], DatanodeInfoWithStorage[127.0.0.1:33959,DS-d9322215-295f-4a40-b488-7f9d6be8d7ad,DISK]]
2024-12-11T20:11:19,659 INFO  [FSHLog-0-hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738-prefix:9fe0640122ec,45805,1733947842477 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:32807,DS-989a9c11-7ce7-48ff-86e3-293ebea318af,DISK], DatanodeInfoWithStorage[127.0.0.1:33959,DS-d9322215-295f-4a40-b488-7f9d6be8d7ad,DISK]]
2024-12-11T20:11:21,863 INFO  [FSHLog-0-hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738-prefix:9fe0640122ec,45805,1733947842477 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:32807,DS-989a9c11-7ce7-48ff-86e3-293ebea318af,DISK], DatanodeInfoWithStorage[127.0.0.1:33959,DS-d9322215-295f-4a40-b488-7f9d6be8d7ad,DISK]]
2024-12-11T20:11:21,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45805 {}] regionserver.HRegion(8855): Flush requested on 5406b284938f2b29683221552ae824d8
2024-12-11T20:11:21,864 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5406b284938f2b29683221552ae824d8 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-11T20:11:22,066 INFO  [FSHLog-0-hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738-prefix:9fe0640122ec,45805,1733947842477 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:32807,DS-989a9c11-7ce7-48ff-86e3-293ebea318af,DISK], DatanodeInfoWithStorage[127.0.0.1:33959,DS-d9322215-295f-4a40-b488-7f9d6be8d7ad,DISK]]
2024-12-11T20:11:22,071 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/default/TestLogRolling-testSlowSyncLogRolling/5406b284938f2b29683221552ae824d8/.tmp/info/ee0502c79f23423aa8cbef358a84d99d is 1080, key is row0008/info:/1733947868815/Put/seqid=0
2024-12-11T20:11:22,080 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33959 is added to blk_1073741840_1016 (size=12509)
2024-12-11T20:11:22,081 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32807 is added to blk_1073741840_1016 (size=12509)
2024-12-11T20:11:22,081 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/default/TestLogRolling-testSlowSyncLogRolling/5406b284938f2b29683221552ae824d8/.tmp/info/ee0502c79f23423aa8cbef358a84d99d
2024-12-11T20:11:22,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/default/TestLogRolling-testSlowSyncLogRolling/5406b284938f2b29683221552ae824d8/.tmp/info/ee0502c79f23423aa8cbef358a84d99d as hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/default/TestLogRolling-testSlowSyncLogRolling/5406b284938f2b29683221552ae824d8/info/ee0502c79f23423aa8cbef358a84d99d
2024-12-11T20:11:22,105 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/default/TestLogRolling-testSlowSyncLogRolling/5406b284938f2b29683221552ae824d8/info/ee0502c79f23423aa8cbef358a84d99d, entries=7, sequenceid=21, filesize=12.2 K
2024-12-11T20:11:22,308 INFO  [FSHLog-0-hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738-prefix:9fe0640122ec,45805,1733947842477 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:32807,DS-989a9c11-7ce7-48ff-86e3-293ebea318af,DISK], DatanodeInfoWithStorage[127.0.0.1:33959,DS-d9322215-295f-4a40-b488-7f9d6be8d7ad,DISK]]
2024-12-11T20:11:22,308 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 5406b284938f2b29683221552ae824d8 in 445ms, sequenceid=21, compaction requested=false
2024-12-11T20:11:22,308 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5406b284938f2b29683221552ae824d8:

2024-12-11T20:11:22,308 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=24.4 K, sizeToCheck=16.0 K
2024-12-11T20:11:22,308 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-11T20:11:22,309 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/default/TestLogRolling-testSlowSyncLogRolling/5406b284938f2b29683221552ae824d8/info/e5656d56806a4366a53b430e3a62224a because midkey is the same as first or last row
2024-12-11T20:11:24,068 INFO  [FSHLog-0-hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738-prefix:9fe0640122ec,45805,1733947842477 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:32807,DS-989a9c11-7ce7-48ff-86e3-293ebea318af,DISK], DatanodeInfoWithStorage[127.0.0.1:33959,DS-d9322215-295f-4a40-b488-7f9d6be8d7ad,DISK]]
2024-12-11T20:11:24,499 INFO  [master/9fe0640122ec:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-12-11T20:11:24,500 INFO  [master/9fe0640122ec:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-12-11T20:11:26,273 INFO  [FSHLog-0-hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738-prefix:9fe0640122ec,45805,1733947842477 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:32807,DS-989a9c11-7ce7-48ff-86e3-293ebea318af,DISK], DatanodeInfoWithStorage[127.0.0.1:33959,DS-d9322215-295f-4a40-b488-7f9d6be8d7ad,DISK]]
2024-12-11T20:11:26,275 WARN  [FSHLog-0-hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738-prefix:9fe0640122ec,45805,1733947842477 {}] wal.AbstractFSWAL(2201): Requesting log roll because we exceeded slow sync threshold; count=8, threshold=5, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:32807,DS-989a9c11-7ce7-48ff-86e3-293ebea318af,DISK], DatanodeInfoWithStorage[127.0.0.1:33959,DS-d9322215-295f-4a40-b488-7f9d6be8d7ad,DISK]]
2024-12-11T20:11:26,276 DEBUG [regionserver/9fe0640122ec:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 9fe0640122ec%2C45805%2C1733947842477:(num 1733947874827) roll requested
2024-12-11T20:11:26,276 INFO  [regionserver/9fe0640122ec:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9fe0640122ec%2C45805%2C1733947842477.1733947886276
2024-12-11T20:11:26,484 INFO  [regionserver/9fe0640122ec:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 206 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:32807,DS-989a9c11-7ce7-48ff-86e3-293ebea318af,DISK], DatanodeInfoWithStorage[127.0.0.1:33959,DS-d9322215-295f-4a40-b488-7f9d6be8d7ad,DISK]]
2024-12-11T20:11:26,485 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:11:26,485 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:11:26,485 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:11:26,485 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:11:26,485 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:11:26,486 INFO  [regionserver/9fe0640122ec:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/WALs/9fe0640122ec,45805,1733947842477/9fe0640122ec%2C45805%2C1733947842477.1733947874827 with entries=8, filesize=7.55 KB; new WAL /user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/WALs/9fe0640122ec,45805,1733947842477/9fe0640122ec%2C45805%2C1733947842477.1733947886276
2024-12-11T20:11:26,487 DEBUG [regionserver/9fe0640122ec:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44867:44867),(127.0.0.1/127.0.0.1:43923:43923)]
2024-12-11T20:11:26,487 DEBUG [regionserver/9fe0640122ec:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/WALs/9fe0640122ec,45805,1733947842477/9fe0640122ec%2C45805%2C1733947842477.1733947874827 is not closed yet, will try archiving it next time
2024-12-11T20:11:26,487 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/WALs/9fe0640122ec,45805,1733947842477/9fe0640122ec%2C45805%2C1733947842477.1733947854749 to hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/oldWALs/9fe0640122ec%2C45805%2C1733947842477.1733947854749
2024-12-11T20:11:26,488 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33959 is added to blk_1073741839_1015 (size=7739)
2024-12-11T20:11:26,488 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32807 is added to blk_1073741839_1015 (size=7739)
2024-12-11T20:11:28,477 INFO  [FSHLog-0-hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738-prefix:9fe0640122ec,45805,1733947842477 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33959,DS-d9322215-295f-4a40-b488-7f9d6be8d7ad,DISK], DatanodeInfoWithStorage[127.0.0.1:32807,DS-989a9c11-7ce7-48ff-86e3-293ebea318af,DISK]]
2024-12-11T20:11:30,076 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 5406b284938f2b29683221552ae824d8, had cached 0 bytes from a total of 25018
2024-12-11T20:11:30,681 INFO  [FSHLog-0-hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738-prefix:9fe0640122ec,45805,1733947842477 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33959,DS-d9322215-295f-4a40-b488-7f9d6be8d7ad,DISK], DatanodeInfoWithStorage[127.0.0.1:32807,DS-989a9c11-7ce7-48ff-86e3-293ebea318af,DISK]]
2024-12-11T20:11:32,885 INFO  [FSHLog-0-hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738-prefix:9fe0640122ec,45805,1733947842477 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33959,DS-d9322215-295f-4a40-b488-7f9d6be8d7ad,DISK], DatanodeInfoWithStorage[127.0.0.1:32807,DS-989a9c11-7ce7-48ff-86e3-293ebea318af,DISK]]
2024-12-11T20:11:35,089 INFO  [FSHLog-0-hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738-prefix:9fe0640122ec,45805,1733947842477 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33959,DS-d9322215-295f-4a40-b488-7f9d6be8d7ad,DISK], DatanodeInfoWithStorage[127.0.0.1:32807,DS-989a9c11-7ce7-48ff-86e3-293ebea318af,DISK]]
2024-12-11T20:11:37,092 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-12-11T20:11:37,092 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9fe0640122ec%2C45805%2C1733947842477.1733947897092
2024-12-11T20:11:40,696 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-11T20:11:42,102 INFO  [Time-limited test {}] wal.AbstractFSWAL(1368): Slow sync cost: 5006 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33959,DS-d9322215-295f-4a40-b488-7f9d6be8d7ad,DISK], DatanodeInfoWithStorage[127.0.0.1:32807,DS-989a9c11-7ce7-48ff-86e3-293ebea318af,DISK]]
2024-12-11T20:11:42,104 WARN  [Time-limited test {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5006 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33959,DS-d9322215-295f-4a40-b488-7f9d6be8d7ad,DISK], DatanodeInfoWithStorage[127.0.0.1:32807,DS-989a9c11-7ce7-48ff-86e3-293ebea318af,DISK]]
2024-12-11T20:11:42,104 DEBUG [regionserver/9fe0640122ec:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 9fe0640122ec%2C45805%2C1733947842477:(num 1733947897092) roll requested
2024-12-11T20:11:42,104 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:11:42,104 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:11:42,104 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:11:42,105 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:11:42,105 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:11:42,105 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/WALs/9fe0640122ec,45805,1733947842477/9fe0640122ec%2C45805%2C1733947842477.1733947886276 with entries=4, filesize=4.63 KB; new WAL /user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/WALs/9fe0640122ec,45805,1733947842477/9fe0640122ec%2C45805%2C1733947842477.1733947897092
2024-12-11T20:11:42,107 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32807 is added to blk_1073741841_1017 (size=4753)
2024-12-11T20:11:42,107 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33959 is added to blk_1073741841_1017 (size=4753)
2024-12-11T20:11:42,114 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44867:44867),(127.0.0.1/127.0.0.1:43923:43923)]
2024-12-11T20:11:42,115 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/WALs/9fe0640122ec,45805,1733947842477/9fe0640122ec%2C45805%2C1733947842477.1733947886276 is not closed yet, will try archiving it next time
2024-12-11T20:11:42,115 INFO  [regionserver/9fe0640122ec:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9fe0640122ec%2C45805%2C1733947842477.1733947902115
2024-12-11T20:11:47,118 INFO  [FSHLog-0-hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738-prefix:9fe0640122ec,45805,1733947842477 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5001 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33959,DS-d9322215-295f-4a40-b488-7f9d6be8d7ad,DISK], DatanodeInfoWithStorage[127.0.0.1:32807,DS-989a9c11-7ce7-48ff-86e3-293ebea318af,DISK]]
2024-12-11T20:11:47,119 WARN  [FSHLog-0-hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738-prefix:9fe0640122ec,45805,1733947842477 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5001 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33959,DS-d9322215-295f-4a40-b488-7f9d6be8d7ad,DISK], DatanodeInfoWithStorage[127.0.0.1:32807,DS-989a9c11-7ce7-48ff-86e3-293ebea318af,DISK]]
2024-12-11T20:11:47,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45805 {}] regionserver.HRegion(8855): Flush requested on 5406b284938f2b29683221552ae824d8
2024-12-11T20:11:47,119 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5406b284938f2b29683221552ae824d8 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-11T20:11:47,123 INFO  [regionserver/9fe0640122ec:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 5006 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33959,DS-d9322215-295f-4a40-b488-7f9d6be8d7ad,DISK], DatanodeInfoWithStorage[127.0.0.1:32807,DS-989a9c11-7ce7-48ff-86e3-293ebea318af,DISK]]
2024-12-11T20:11:47,123 WARN  [regionserver/9fe0640122ec:0.logRoller {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5006 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33959,DS-d9322215-295f-4a40-b488-7f9d6be8d7ad,DISK], DatanodeInfoWithStorage[127.0.0.1:32807,DS-989a9c11-7ce7-48ff-86e3-293ebea318af,DISK]]
2024-12-11T20:11:49,120 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-12-11T20:11:52,121 INFO  [FSHLog-0-hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738-prefix:9fe0640122ec,45805,1733947842477 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5001 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33959,DS-d9322215-295f-4a40-b488-7f9d6be8d7ad,DISK], DatanodeInfoWithStorage[127.0.0.1:32807,DS-989a9c11-7ce7-48ff-86e3-293ebea318af,DISK]]
2024-12-11T20:11:52,121 WARN  [FSHLog-0-hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738-prefix:9fe0640122ec,45805,1733947842477 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5001 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33959,DS-d9322215-295f-4a40-b488-7f9d6be8d7ad,DISK], DatanodeInfoWithStorage[127.0.0.1:32807,DS-989a9c11-7ce7-48ff-86e3-293ebea318af,DISK]]
2024-12-11T20:11:52,122 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:11:52,122 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:11:52,122 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:11:52,122 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:11:52,122 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:11:52,123 INFO  [regionserver/9fe0640122ec:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/WALs/9fe0640122ec,45805,1733947842477/9fe0640122ec%2C45805%2C1733947842477.1733947897092 with entries=2, filesize=1.52 KB; new WAL /user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/WALs/9fe0640122ec,45805,1733947842477/9fe0640122ec%2C45805%2C1733947842477.1733947902115
2024-12-11T20:11:52,124 DEBUG [regionserver/9fe0640122ec:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44867:44867),(127.0.0.1/127.0.0.1:43923:43923)]
2024-12-11T20:11:52,124 DEBUG [regionserver/9fe0640122ec:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/WALs/9fe0640122ec,45805,1733947842477/9fe0640122ec%2C45805%2C1733947842477.1733947897092 is not closed yet, will try archiving it next time
2024-12-11T20:11:52,124 DEBUG [regionserver/9fe0640122ec:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 9fe0640122ec%2C45805%2C1733947842477:(num 1733947902115) roll requested
2024-12-11T20:11:52,124 INFO  [regionserver/9fe0640122ec:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9fe0640122ec%2C45805%2C1733947842477.1733947912124
2024-12-11T20:11:52,127 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/default/TestLogRolling-testSlowSyncLogRolling/5406b284938f2b29683221552ae824d8/.tmp/info/5a62669add3c431e830e6e0fd5ce6823 is 1080, key is row0015/info:/1733947883866/Put/seqid=0
2024-12-11T20:11:52,128 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32807 is added to blk_1073741842_1018 (size=1569)
2024-12-11T20:11:52,129 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33959 is added to blk_1073741842_1018 (size=1569)
2024-12-11T20:11:52,134 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32807 is added to blk_1073741844_1020 (size=12509)
2024-12-11T20:11:52,135 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33959 is added to blk_1073741844_1020 (size=12509)
2024-12-11T20:11:52,135 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/default/TestLogRolling-testSlowSyncLogRolling/5406b284938f2b29683221552ae824d8/.tmp/info/5a62669add3c431e830e6e0fd5ce6823
2024-12-11T20:11:52,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/default/TestLogRolling-testSlowSyncLogRolling/5406b284938f2b29683221552ae824d8/.tmp/info/5a62669add3c431e830e6e0fd5ce6823 as hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/default/TestLogRolling-testSlowSyncLogRolling/5406b284938f2b29683221552ae824d8/info/5a62669add3c431e830e6e0fd5ce6823
2024-12-11T20:11:52,156 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/default/TestLogRolling-testSlowSyncLogRolling/5406b284938f2b29683221552ae824d8/info/5a62669add3c431e830e6e0fd5ce6823, entries=7, sequenceid=31, filesize=12.2 K
2024-12-11T20:11:57,132 INFO  [regionserver/9fe0640122ec:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 5005 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33959,DS-d9322215-295f-4a40-b488-7f9d6be8d7ad,DISK], DatanodeInfoWithStorage[127.0.0.1:32807,DS-989a9c11-7ce7-48ff-86e3-293ebea318af,DISK]]
2024-12-11T20:11:57,132 WARN  [regionserver/9fe0640122ec:0.logRoller {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5005 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33959,DS-d9322215-295f-4a40-b488-7f9d6be8d7ad,DISK], DatanodeInfoWithStorage[127.0.0.1:32807,DS-989a9c11-7ce7-48ff-86e3-293ebea318af,DISK]]
2024-12-11T20:11:57,158 INFO  [FSHLog-0-hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738-prefix:9fe0640122ec,45805,1733947842477 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5001 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33959,DS-d9322215-295f-4a40-b488-7f9d6be8d7ad,DISK], DatanodeInfoWithStorage[127.0.0.1:32807,DS-989a9c11-7ce7-48ff-86e3-293ebea318af,DISK]]
2024-12-11T20:11:57,158 WARN  [FSHLog-0-hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738-prefix:9fe0640122ec,45805,1733947842477 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5001 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33959,DS-d9322215-295f-4a40-b488-7f9d6be8d7ad,DISK], DatanodeInfoWithStorage[127.0.0.1:32807,DS-989a9c11-7ce7-48ff-86e3-293ebea318af,DISK]]
2024-12-11T20:11:57,158 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 5406b284938f2b29683221552ae824d8 in 10039ms, sequenceid=31, compaction requested=true
2024-12-11T20:11:57,158 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:11:57,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5406b284938f2b29683221552ae824d8:

2024-12-11T20:11:57,158 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:11:57,158 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=36.6 K, sizeToCheck=16.0 K
2024-12-11T20:11:57,158 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-11T20:11:57,158 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:11:57,158 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/default/TestLogRolling-testSlowSyncLogRolling/5406b284938f2b29683221552ae824d8/info/e5656d56806a4366a53b430e3a62224a because midkey is the same as first or last row
2024-12-11T20:11:57,159 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:11:57,159 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:11:57,159 INFO  [regionserver/9fe0640122ec:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/WALs/9fe0640122ec,45805,1733947842477/9fe0640122ec%2C45805%2C1733947842477.1733947902115 with entries=1, filesize=430 B; new WAL /user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/WALs/9fe0640122ec,45805,1733947842477/9fe0640122ec%2C45805%2C1733947842477.1733947912124
2024-12-11T20:11:57,160 DEBUG [regionserver/9fe0640122ec:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43923:43923),(127.0.0.1/127.0.0.1:44867:44867)]
2024-12-11T20:11:57,160 DEBUG [regionserver/9fe0640122ec:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/WALs/9fe0640122ec,45805,1733947842477/9fe0640122ec%2C45805%2C1733947842477.1733947902115 is not closed yet, will try archiving it next time
2024-12-11T20:11:57,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5406b284938f2b29683221552ae824d8:info, priority=-2147483648, current under compaction store size is 1
2024-12-11T20:11:57,160 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/WALs/9fe0640122ec,45805,1733947842477/9fe0640122ec%2C45805%2C1733947842477.1733947874827 to hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/oldWALs/9fe0640122ec%2C45805%2C1733947842477.1733947874827
2024-12-11T20:11:57,161 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9fe0640122ec%2C45805%2C1733947842477.1733947917160
2024-12-11T20:11:57,161 DEBUG [regionserver/9fe0640122ec:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 9fe0640122ec%2C45805%2C1733947842477:(num 1733947917160) roll requested
2024-12-11T20:11:57,163 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33959 is added to blk_1073741843_1019 (size=438)
2024-12-11T20:11:57,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-11T20:11:57,163 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/WALs/9fe0640122ec,45805,1733947842477/9fe0640122ec%2C45805%2C1733947842477.1733947886276 to hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/oldWALs/9fe0640122ec%2C45805%2C1733947842477.1733947886276
2024-12-11T20:11:57,163 DEBUG [RS:0;9fe0640122ec:45805-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-11T20:11:57,163 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32807 is added to blk_1073741843_1019 (size=438)
2024-12-11T20:11:57,165 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/WALs/9fe0640122ec,45805,1733947842477/9fe0640122ec%2C45805%2C1733947842477.1733947897092 to hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/oldWALs/9fe0640122ec%2C45805%2C1733947842477.1733947897092
2024-12-11T20:11:57,166 DEBUG [RS:0;9fe0640122ec:45805-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 37527 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-11T20:11:57,168 DEBUG [RS:0;9fe0640122ec:45805-shortCompactions-0 {}] regionserver.HStore(1541): 5406b284938f2b29683221552ae824d8/info is initiating minor compaction (all files)
2024-12-11T20:11:57,168 INFO  [RS:0;9fe0640122ec:45805-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5406b284938f2b29683221552ae824d8/info in TestLogRolling-testSlowSyncLogRolling,,1733947844623.5406b284938f2b29683221552ae824d8.
2024-12-11T20:11:57,168 INFO  [RS:0;9fe0640122ec:45805-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/default/TestLogRolling-testSlowSyncLogRolling/5406b284938f2b29683221552ae824d8/info/e5656d56806a4366a53b430e3a62224a, hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/default/TestLogRolling-testSlowSyncLogRolling/5406b284938f2b29683221552ae824d8/info/ee0502c79f23423aa8cbef358a84d99d, hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/default/TestLogRolling-testSlowSyncLogRolling/5406b284938f2b29683221552ae824d8/info/5a62669add3c431e830e6e0fd5ce6823] into tmpdir=hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/default/TestLogRolling-testSlowSyncLogRolling/5406b284938f2b29683221552ae824d8/.tmp, totalSize=36.6 K
2024-12-11T20:11:57,170 DEBUG [RS:0;9fe0640122ec:45805-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5656d56806a4366a53b430e3a62224a, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733947854773
2024-12-11T20:11:57,170 DEBUG [RS:0;9fe0640122ec:45805-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee0502c79f23423aa8cbef358a84d99d, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=21, earliestPutTs=1733947868815
2024-12-11T20:11:57,171 DEBUG [RS:0;9fe0640122ec:45805-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a62669add3c431e830e6e0fd5ce6823, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=31, earliestPutTs=1733947883866
2024-12-11T20:11:57,174 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:11:57,174 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:11:57,174 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:11:57,174 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:11:57,174 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:11:57,174 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/WALs/9fe0640122ec,45805,1733947842477/9fe0640122ec%2C45805%2C1733947842477.1733947912124 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/WALs/9fe0640122ec,45805,1733947842477/9fe0640122ec%2C45805%2C1733947842477.1733947917160
2024-12-11T20:11:57,178 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33959 is added to blk_1073741845_1021 (size=93)
2024-12-11T20:11:57,179 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32807 is added to blk_1073741845_1021 (size=93)
2024-12-11T20:11:57,180 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/WALs/9fe0640122ec,45805,1733947842477/9fe0640122ec%2C45805%2C1733947842477.1733947902115 is not closed yet, will try archiving it next time
2024-12-11T20:11:57,180 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/WALs/9fe0640122ec,45805,1733947842477/9fe0640122ec%2C45805%2C1733947842477.1733947912124 to hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/oldWALs/9fe0640122ec%2C45805%2C1733947842477.1733947912124
2024-12-11T20:11:57,185 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44867:44867),(127.0.0.1/127.0.0.1:43923:43923)]
2024-12-11T20:11:57,185 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/WALs/9fe0640122ec,45805,1733947842477/9fe0640122ec%2C45805%2C1733947842477.1733947902115 is not closed yet, will try archiving it next time
2024-12-11T20:11:57,185 INFO  [regionserver/9fe0640122ec:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9fe0640122ec%2C45805%2C1733947842477.1733947917185
2024-12-11T20:11:57,194 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:11:57,194 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:11:57,194 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:11:57,194 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:11:57,195 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:11:57,195 INFO  [regionserver/9fe0640122ec:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/WALs/9fe0640122ec,45805,1733947842477/9fe0640122ec%2C45805%2C1733947842477.1733947917160 with entries=1, filesize=1.22 KB; new WAL /user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/WALs/9fe0640122ec,45805,1733947842477/9fe0640122ec%2C45805%2C1733947842477.1733947917185
2024-12-11T20:11:57,197 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32807 is added to blk_1073741846_1022 (size=1258)
2024-12-11T20:11:57,197 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33959 is added to blk_1073741846_1022 (size=1258)
2024-12-11T20:11:57,198 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/WALs/9fe0640122ec,45805,1733947842477/9fe0640122ec%2C45805%2C1733947842477.1733947902115 is not closed yet, will try archiving it next time
2024-12-11T20:11:57,201 DEBUG [regionserver/9fe0640122ec:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44867:44867),(127.0.0.1/127.0.0.1:43923:43923)]
2024-12-11T20:11:57,201 DEBUG [regionserver/9fe0640122ec:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/WALs/9fe0640122ec,45805,1733947842477/9fe0640122ec%2C45805%2C1733947842477.1733947902115 is not closed yet, will try archiving it next time
2024-12-11T20:11:57,209 INFO  [RS:0;9fe0640122ec:45805-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5406b284938f2b29683221552ae824d8#info#compaction#3 average throughput is 7.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-11T20:11:57,210 DEBUG [RS:0;9fe0640122ec:45805-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/default/TestLogRolling-testSlowSyncLogRolling/5406b284938f2b29683221552ae824d8/.tmp/info/f82fa33433824f2584affa712fb12401 is 1080, key is row0001/info:/1733947854773/Put/seqid=0
2024-12-11T20:11:57,217 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33959 is added to blk_1073741848_1024 (size=27710)
2024-12-11T20:11:57,217 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32807 is added to blk_1073741848_1024 (size=27710)
2024-12-11T20:11:57,230 DEBUG [RS:0;9fe0640122ec:45805-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/default/TestLogRolling-testSlowSyncLogRolling/5406b284938f2b29683221552ae824d8/.tmp/info/f82fa33433824f2584affa712fb12401 as hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/default/TestLogRolling-testSlowSyncLogRolling/5406b284938f2b29683221552ae824d8/info/f82fa33433824f2584affa712fb12401
2024-12-11T20:11:57,251 INFO  [RS:0;9fe0640122ec:45805-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5406b284938f2b29683221552ae824d8/info of 5406b284938f2b29683221552ae824d8 into f82fa33433824f2584affa712fb12401(size=27.1 K), total size for store is 27.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-11T20:11:57,251 DEBUG [RS:0;9fe0640122ec:45805-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5406b284938f2b29683221552ae824d8:

2024-12-11T20:11:57,253 INFO  [RS:0;9fe0640122ec:45805-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testSlowSyncLogRolling,,1733947844623.5406b284938f2b29683221552ae824d8., storeName=5406b284938f2b29683221552ae824d8/info, priority=13, startTime=1733947917160; duration=0sec
2024-12-11T20:11:57,254 DEBUG [RS:0;9fe0640122ec:45805-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K
2024-12-11T20:11:57,254 DEBUG [RS:0;9fe0640122ec:45805-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-11T20:11:57,254 DEBUG [RS:0;9fe0640122ec:45805-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/default/TestLogRolling-testSlowSyncLogRolling/5406b284938f2b29683221552ae824d8/info/f82fa33433824f2584affa712fb12401 because midkey is the same as first or last row
2024-12-11T20:11:57,254 DEBUG [RS:0;9fe0640122ec:45805-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K
2024-12-11T20:11:57,255 DEBUG [RS:0;9fe0640122ec:45805-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-11T20:11:57,255 DEBUG [RS:0;9fe0640122ec:45805-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/default/TestLogRolling-testSlowSyncLogRolling/5406b284938f2b29683221552ae824d8/info/f82fa33433824f2584affa712fb12401 because midkey is the same as first or last row
2024-12-11T20:11:57,255 DEBUG [RS:0;9fe0640122ec:45805-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K
2024-12-11T20:11:57,255 DEBUG [RS:0;9fe0640122ec:45805-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-11T20:11:57,255 DEBUG [RS:0;9fe0640122ec:45805-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/default/TestLogRolling-testSlowSyncLogRolling/5406b284938f2b29683221552ae824d8/info/f82fa33433824f2584affa712fb12401 because midkey is the same as first or last row
2024-12-11T20:11:57,255 DEBUG [RS:0;9fe0640122ec:45805-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-11T20:11:57,255 DEBUG [RS:0;9fe0640122ec:45805-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5406b284938f2b29683221552ae824d8:info
2024-12-11T20:11:57,564 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/WALs/9fe0640122ec,45805,1733947842477/9fe0640122ec%2C45805%2C1733947842477.1733947902115 to hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/oldWALs/9fe0640122ec%2C45805%2C1733947842477.1733947902115
2024-12-11T20:12:09,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45805 {}] regionserver.HRegion(8855): Flush requested on 5406b284938f2b29683221552ae824d8
2024-12-11T20:12:09,210 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5406b284938f2b29683221552ae824d8 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-11T20:12:09,218 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/default/TestLogRolling-testSlowSyncLogRolling/5406b284938f2b29683221552ae824d8/.tmp/info/1e27624c490b40a2a01889cbc99ae07a is 1080, key is row0022/info:/1733947917186/Put/seqid=0
2024-12-11T20:12:09,225 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33959 is added to blk_1073741849_1025 (size=12509)
2024-12-11T20:12:09,225 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32807 is added to blk_1073741849_1025 (size=12509)
2024-12-11T20:12:09,226 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/default/TestLogRolling-testSlowSyncLogRolling/5406b284938f2b29683221552ae824d8/.tmp/info/1e27624c490b40a2a01889cbc99ae07a
2024-12-11T20:12:09,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/default/TestLogRolling-testSlowSyncLogRolling/5406b284938f2b29683221552ae824d8/.tmp/info/1e27624c490b40a2a01889cbc99ae07a as hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/default/TestLogRolling-testSlowSyncLogRolling/5406b284938f2b29683221552ae824d8/info/1e27624c490b40a2a01889cbc99ae07a
2024-12-11T20:12:09,244 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/default/TestLogRolling-testSlowSyncLogRolling/5406b284938f2b29683221552ae824d8/info/1e27624c490b40a2a01889cbc99ae07a, entries=7, sequenceid=42, filesize=12.2 K
2024-12-11T20:12:09,245 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 5406b284938f2b29683221552ae824d8 in 36ms, sequenceid=42, compaction requested=false
2024-12-11T20:12:09,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5406b284938f2b29683221552ae824d8:

2024-12-11T20:12:09,245 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=39.3 K, sizeToCheck=16.0 K
2024-12-11T20:12:09,245 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-11T20:12:09,245 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/default/TestLogRolling-testSlowSyncLogRolling/5406b284938f2b29683221552ae824d8/info/f82fa33433824f2584affa712fb12401 because midkey is the same as first or last row
2024-12-11T20:12:10,697 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-11T20:12:15,077 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 5406b284938f2b29683221552ae824d8, had cached 0 bytes from a total of 40219
2024-12-11T20:12:17,221 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-12-11T20:12:17,222 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-11T20:12:17,222 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-11T20:12:17,227 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T20:12:17,227 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T20:12:17,227 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-12-11T20:12:17,228 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-11T20:12:17,228 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1904156029, stopped=false
2024-12-11T20:12:17,228 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=9fe0640122ec,35671,1733947841689
2024-12-11T20:12:17,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35671-0x100caf66b7a0000, quorum=127.0.0.1:58160, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-11T20:12:17,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45805-0x100caf66b7a0001, quorum=127.0.0.1:58160, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-11T20:12:17,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35671-0x100caf66b7a0000, quorum=127.0.0.1:58160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:12:17,230 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-11T20:12:17,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45805-0x100caf66b7a0001, quorum=127.0.0.1:58160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:12:17,230 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-11T20:12:17,231 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-11T20:12:17,231 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T20:12:17,231 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45805-0x100caf66b7a0001, quorum=127.0.0.1:58160, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-11T20:12:17,231 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9fe0640122ec,45805,1733947842477' *****
2024-12-11T20:12:17,231 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35671-0x100caf66b7a0000, quorum=127.0.0.1:58160, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-11T20:12:17,231 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-12-11T20:12:17,232 INFO  [RS:0;9fe0640122ec:45805 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-11T20:12:17,232 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-12-11T20:12:17,232 INFO  [RS:0;9fe0640122ec:45805 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-11T20:12:17,232 INFO  [RS:0;9fe0640122ec:45805 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-11T20:12:17,233 INFO  [RS:0;9fe0640122ec:45805 {}] regionserver.HRegionServer(3091): Received CLOSE for 5406b284938f2b29683221552ae824d8
2024-12-11T20:12:17,233 INFO  [RS:0;9fe0640122ec:45805 {}] regionserver.HRegionServer(959): stopping server 9fe0640122ec,45805,1733947842477
2024-12-11T20:12:17,233 INFO  [RS:0;9fe0640122ec:45805 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-11T20:12:17,233 INFO  [RS:0;9fe0640122ec:45805 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;9fe0640122ec:45805.
2024-12-11T20:12:17,233 DEBUG [RS:0;9fe0640122ec:45805 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-11T20:12:17,233 DEBUG [RS:0;9fe0640122ec:45805 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T20:12:17,234 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5406b284938f2b29683221552ae824d8, disabling compactions & flushes
2024-12-11T20:12:17,234 INFO  [RS:0;9fe0640122ec:45805 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-11T20:12:17,234 INFO  [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testSlowSyncLogRolling,,1733947844623.5406b284938f2b29683221552ae824d8.
2024-12-11T20:12:17,234 INFO  [RS:0;9fe0640122ec:45805 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-11T20:12:17,234 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testSlowSyncLogRolling,,1733947844623.5406b284938f2b29683221552ae824d8.
2024-12-11T20:12:17,234 INFO  [RS:0;9fe0640122ec:45805 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-11T20:12:17,234 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1733947844623.5406b284938f2b29683221552ae824d8. after waiting 0 ms
2024-12-11T20:12:17,234 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1733947844623.5406b284938f2b29683221552ae824d8.
2024-12-11T20:12:17,234 INFO  [RS:0;9fe0640122ec:45805 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-12-11T20:12:17,234 INFO  [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 5406b284938f2b29683221552ae824d8 1/1 column families, dataSize=3.15 KB heapSize=3.63 KB
2024-12-11T20:12:17,234 INFO  [RS:0;9fe0640122ec:45805 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-12-11T20:12:17,234 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-11T20:12:17,234 DEBUG [RS:0;9fe0640122ec:45805 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 5406b284938f2b29683221552ae824d8=TestLogRolling-testSlowSyncLogRolling,,1733947844623.5406b284938f2b29683221552ae824d8.}
2024-12-11T20:12:17,235 INFO  [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-11T20:12:17,235 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-11T20:12:17,235 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-11T20:12:17,235 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-11T20:12:17,235 DEBUG [RS:0;9fe0640122ec:45805 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 5406b284938f2b29683221552ae824d8
2024-12-11T20:12:17,235 INFO  [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.65 KB heapSize=3.67 KB
2024-12-11T20:12:17,242 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/default/TestLogRolling-testSlowSyncLogRolling/5406b284938f2b29683221552ae824d8/.tmp/info/86f593feae9749a1aeeafe3bdc432cf7 is 1080, key is row0029/info:/1733947931212/Put/seqid=0
2024-12-11T20:12:17,250 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32807 is added to blk_1073741850_1026 (size=8193)
2024-12-11T20:12:17,251 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33959 is added to blk_1073741850_1026 (size=8193)
2024-12-11T20:12:17,251 INFO  [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.15 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/default/TestLogRolling-testSlowSyncLogRolling/5406b284938f2b29683221552ae824d8/.tmp/info/86f593feae9749a1aeeafe3bdc432cf7
2024-12-11T20:12:17,261 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/hbase/meta/1588230740/.tmp/info/b478204b078d4771a3fa967569355de9 is 195, key is TestLogRolling-testSlowSyncLogRolling,,1733947844623.5406b284938f2b29683221552ae824d8./info:regioninfo/1733947845126/Put/seqid=0
2024-12-11T20:12:17,261 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/default/TestLogRolling-testSlowSyncLogRolling/5406b284938f2b29683221552ae824d8/.tmp/info/86f593feae9749a1aeeafe3bdc432cf7 as hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/default/TestLogRolling-testSlowSyncLogRolling/5406b284938f2b29683221552ae824d8/info/86f593feae9749a1aeeafe3bdc432cf7
2024-12-11T20:12:17,271 INFO  [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/default/TestLogRolling-testSlowSyncLogRolling/5406b284938f2b29683221552ae824d8/info/86f593feae9749a1aeeafe3bdc432cf7, entries=3, sequenceid=48, filesize=8.0 K
2024-12-11T20:12:17,272 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33959 is added to blk_1073741851_1027 (size=7016)
2024-12-11T20:12:17,272 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32807 is added to blk_1073741851_1027 (size=7016)
2024-12-11T20:12:17,273 INFO  [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 5406b284938f2b29683221552ae824d8 in 38ms, sequenceid=48, compaction requested=true
2024-12-11T20:12:17,273 INFO  [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.45 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/hbase/meta/1588230740/.tmp/info/b478204b078d4771a3fa967569355de9
2024-12-11T20:12:17,273 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733947844623.5406b284938f2b29683221552ae824d8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/default/TestLogRolling-testSlowSyncLogRolling/5406b284938f2b29683221552ae824d8/info/e5656d56806a4366a53b430e3a62224a, hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/default/TestLogRolling-testSlowSyncLogRolling/5406b284938f2b29683221552ae824d8/info/ee0502c79f23423aa8cbef358a84d99d, hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/default/TestLogRolling-testSlowSyncLogRolling/5406b284938f2b29683221552ae824d8/info/5a62669add3c431e830e6e0fd5ce6823] to archive
2024-12-11T20:12:17,277 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733947844623.5406b284938f2b29683221552ae824d8.-1 {}] backup.HFileArchiver(363): Archiving compacted files.
2024-12-11T20:12:17,282 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/default/TestLogRolling-testSlowSyncLogRolling/5406b284938f2b29683221552ae824d8/info/e5656d56806a4366a53b430e3a62224a to hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/archive/data/default/TestLogRolling-testSlowSyncLogRolling/5406b284938f2b29683221552ae824d8/info/e5656d56806a4366a53b430e3a62224a
2024-12-11T20:12:17,282 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/default/TestLogRolling-testSlowSyncLogRolling/5406b284938f2b29683221552ae824d8/info/ee0502c79f23423aa8cbef358a84d99d to hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/archive/data/default/TestLogRolling-testSlowSyncLogRolling/5406b284938f2b29683221552ae824d8/info/ee0502c79f23423aa8cbef358a84d99d
2024-12-11T20:12:17,282 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/default/TestLogRolling-testSlowSyncLogRolling/5406b284938f2b29683221552ae824d8/info/5a62669add3c431e830e6e0fd5ce6823 to hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/archive/data/default/TestLogRolling-testSlowSyncLogRolling/5406b284938f2b29683221552ae824d8/info/5a62669add3c431e830e6e0fd5ce6823
2024-12-11T20:12:17,299 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/hbase/meta/1588230740/.tmp/ns/3d44c5fd7e574a5fa7daf107da2013cc is 43, key is default/ns:d/1733947844395/Put/seqid=0
2024-12-11T20:12:17,297 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733947844623.5406b284938f2b29683221552ae824d8.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=9fe0640122ec:35671 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 16 more
2024-12-11T20:12:17,302 WARN  [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733947844623.5406b284938f2b29683221552ae824d8.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [e5656d56806a4366a53b430e3a62224a=12509, ee0502c79f23423aa8cbef358a84d99d=12509, 5a62669add3c431e830e6e0fd5ce6823=12509]
2024-12-11T20:12:17,305 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33959 is added to blk_1073741852_1028 (size=5153)
2024-12-11T20:12:17,306 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32807 is added to blk_1073741852_1028 (size=5153)
2024-12-11T20:12:17,306 INFO  [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/hbase/meta/1588230740/.tmp/ns/3d44c5fd7e574a5fa7daf107da2013cc
2024-12-11T20:12:17,308 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/default/TestLogRolling-testSlowSyncLogRolling/5406b284938f2b29683221552ae824d8/recovered.edits/51.seqid, newMaxSeqId=51, maxSeqId=1
2024-12-11T20:12:17,311 INFO  [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testSlowSyncLogRolling,,1733947844623.5406b284938f2b29683221552ae824d8.
2024-12-11T20:12:17,311 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5406b284938f2b29683221552ae824d8:
Waiting for close lock at 1733947937233Running coprocessor pre-close hooks at 1733947937233Disabling compacts and flushes for region at 1733947937233Disabling writes for close at 1733947937234 (+1 ms)Obtaining lock to block concurrent updates at 1733947937234Preparing flush snapshotting stores in 5406b284938f2b29683221552ae824d8 at 1733947937234Finished memstore snapshotting TestLogRolling-testSlowSyncLogRolling,,1733947844623.5406b284938f2b29683221552ae824d8., syncing WAL and waiting on mvcc, flushsize=dataSize=3228, getHeapSize=3696, getOffHeapSize=0, getCellsCount=3 at 1733947937234Flushing stores of TestLogRolling-testSlowSyncLogRolling,,1733947844623.5406b284938f2b29683221552ae824d8. at 1733947937236 (+2 ms)Flushing 5406b284938f2b29683221552ae824d8/info: creating writer at 1733947937236Flushing 5406b284938f2b29683221552ae824d8/info: appending metadata at 1733947937242 (+6 ms)Flushing 5406b284938f2b29683221552ae824d8/info: closing flushed file at 1733947937242Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@208a5a59: reopening flushed file at 1733947937260 (+18 ms)Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 5406b284938f2b29683221552ae824d8 in 38ms, sequenceid=48, compaction requested=true at 1733947937273 (+13 ms)Writing region close event to WAL at 1733947937303 (+30 ms)Running coprocessor post-close hooks at 1733947937309 (+6 ms)Closed at 1733947937311 (+2 ms)
2024-12-11T20:12:17,312 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testSlowSyncLogRolling,,1733947844623.5406b284938f2b29683221552ae824d8.
2024-12-11T20:12:17,332 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/hbase/meta/1588230740/.tmp/table/d0ab87202c854489b6524a5947579028 is 73, key is TestLogRolling-testSlowSyncLogRolling/table:state/1733947845145/Put/seqid=0
2024-12-11T20:12:17,338 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33959 is added to blk_1073741853_1029 (size=5396)
2024-12-11T20:12:17,338 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32807 is added to blk_1073741853_1029 (size=5396)
2024-12-11T20:12:17,338 INFO  [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=138 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/hbase/meta/1588230740/.tmp/table/d0ab87202c854489b6524a5947579028
2024-12-11T20:12:17,347 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/hbase/meta/1588230740/.tmp/info/b478204b078d4771a3fa967569355de9 as hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/hbase/meta/1588230740/info/b478204b078d4771a3fa967569355de9
2024-12-11T20:12:17,356 INFO  [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/hbase/meta/1588230740/info/b478204b078d4771a3fa967569355de9, entries=10, sequenceid=11, filesize=6.9 K
2024-12-11T20:12:17,357 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/hbase/meta/1588230740/.tmp/ns/3d44c5fd7e574a5fa7daf107da2013cc as hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/hbase/meta/1588230740/ns/3d44c5fd7e574a5fa7daf107da2013cc
2024-12-11T20:12:17,366 INFO  [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/hbase/meta/1588230740/ns/3d44c5fd7e574a5fa7daf107da2013cc, entries=2, sequenceid=11, filesize=5.0 K
2024-12-11T20:12:17,367 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/hbase/meta/1588230740/.tmp/table/d0ab87202c854489b6524a5947579028 as hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/hbase/meta/1588230740/table/d0ab87202c854489b6524a5947579028
2024-12-11T20:12:17,374 INFO  [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/hbase/meta/1588230740/table/d0ab87202c854489b6524a5947579028, entries=2, sequenceid=11, filesize=5.3 K
2024-12-11T20:12:17,376 INFO  [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.65 KB/1692, heapSize ~3.38 KB/3456, currentSize=0 B/0 for 1588230740 in 141ms, sequenceid=11, compaction requested=false
2024-12-11T20:12:17,382 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-12-11T20:12:17,383 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-11T20:12:17,383 INFO  [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-11T20:12:17,383 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733947937234Running coprocessor pre-close hooks at 1733947937234Disabling compacts and flushes for region at 1733947937234Disabling writes for close at 1733947937235 (+1 ms)Obtaining lock to block concurrent updates at 1733947937235Preparing flush snapshotting stores in 1588230740 at 1733947937235Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1692, getHeapSize=3696, getOffHeapSize=0, getCellsCount=14 at 1733947937236 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733947937236Flushing 1588230740/info: creating writer at 1733947937237 (+1 ms)Flushing 1588230740/info: appending metadata at 1733947937261 (+24 ms)Flushing 1588230740/info: closing flushed file at 1733947937261Flushing 1588230740/ns: creating writer at 1733947937282 (+21 ms)Flushing 1588230740/ns: appending metadata at 1733947937298 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1733947937298Flushing 1588230740/table: creating writer at 1733947937315 (+17 ms)Flushing 1588230740/table: appending metadata at 1733947937331 (+16 ms)Flushing 1588230740/table: closing flushed file at 1733947937331Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3933cae3: reopening flushed file at 1733947937345 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b8abdb7: reopening flushed file at 1733947937356 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@28cba10e: reopening flushed file at 1733947937366 (+10 ms)Finished flush of dataSize ~1.65 KB/1692, heapSize ~3.38 KB/3456, currentSize=0 B/0 for 1588230740 in 141ms, sequenceid=11, compaction requested=false at 1733947937376 (+10 ms)Writing region close event to WAL at 1733947937377 (+1 ms)Running coprocessor post-close hooks at 1733947937383 (+6 ms)Closed at 1733947937383
2024-12-11T20:12:17,383 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-11T20:12:17,435 INFO  [RS:0;9fe0640122ec:45805 {}] regionserver.HRegionServer(976): stopping server 9fe0640122ec,45805,1733947842477; all regions closed.
2024-12-11T20:12:17,437 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:12:17,437 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:12:17,437 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:12:17,437 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:12:17,438 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:12:17,441 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32807 is added to blk_1073741834_1010 (size=3066)
2024-12-11T20:12:17,441 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33959 is added to blk_1073741834_1010 (size=3066)
2024-12-11T20:12:17,445 DEBUG [RS:0;9fe0640122ec:45805 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/oldWALs
2024-12-11T20:12:17,445 INFO  [RS:0;9fe0640122ec:45805 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 9fe0640122ec%2C45805%2C1733947842477.meta:.meta(num 1733947844235)
2024-12-11T20:12:17,445 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:12:17,445 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:12:17,446 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:12:17,446 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:12:17,446 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:12:17,448 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32807 is added to blk_1073741847_1023 (size=12695)
2024-12-11T20:12:17,449 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33959 is added to blk_1073741847_1023 (size=12695)
2024-12-11T20:12:17,453 DEBUG [RS:0;9fe0640122ec:45805 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/oldWALs
2024-12-11T20:12:17,453 INFO  [RS:0;9fe0640122ec:45805 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 9fe0640122ec%2C45805%2C1733947842477:(num 1733947917185)
2024-12-11T20:12:17,453 DEBUG [RS:0;9fe0640122ec:45805 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T20:12:17,453 INFO  [RS:0;9fe0640122ec:45805 {}] regionserver.LeaseManager(133): Closed leases
2024-12-11T20:12:17,454 INFO  [RS:0;9fe0640122ec:45805 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-11T20:12:17,454 INFO  [RS:0;9fe0640122ec:45805 {}] hbase.ChoreService(370): Chore service for: regionserver/9fe0640122ec:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-12-11T20:12:17,454 INFO  [RS:0;9fe0640122ec:45805 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-11T20:12:17,454 INFO  [regionserver/9fe0640122ec:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-11T20:12:17,455 INFO  [RS:0;9fe0640122ec:45805 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45805
2024-12-11T20:12:17,461 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45805-0x100caf66b7a0001, quorum=127.0.0.1:58160, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9fe0640122ec,45805,1733947842477
2024-12-11T20:12:17,461 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35671-0x100caf66b7a0000, quorum=127.0.0.1:58160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-11T20:12:17,461 INFO  [RS:0;9fe0640122ec:45805 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-11T20:12:17,463 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9fe0640122ec,45805,1733947842477]
2024-12-11T20:12:17,465 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9fe0640122ec,45805,1733947842477 already deleted, retry=false
2024-12-11T20:12:17,465 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 9fe0640122ec,45805,1733947842477 expired; onlineServers=0
2024-12-11T20:12:17,465 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '9fe0640122ec,35671,1733947841689' *****
2024-12-11T20:12:17,465 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-11T20:12:17,465 INFO  [M:0;9fe0640122ec:35671 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-11T20:12:17,466 INFO  [M:0;9fe0640122ec:35671 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-11T20:12:17,466 DEBUG [M:0;9fe0640122ec:35671 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-11T20:12:17,466 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-11T20:12:17,466 DEBUG [M:0;9fe0640122ec:35671 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-11T20:12:17,466 DEBUG [master/9fe0640122ec:0:becomeActiveMaster-HFileCleaner.small.0-1733947843483 {}] cleaner.HFileCleaner(306): Exit Thread[master/9fe0640122ec:0:becomeActiveMaster-HFileCleaner.small.0-1733947843483,5,FailOnTimeoutGroup]
2024-12-11T20:12:17,466 DEBUG [master/9fe0640122ec:0:becomeActiveMaster-HFileCleaner.large.0-1733947843482 {}] cleaner.HFileCleaner(306): Exit Thread[master/9fe0640122ec:0:becomeActiveMaster-HFileCleaner.large.0-1733947843482,5,FailOnTimeoutGroup]
2024-12-11T20:12:17,466 INFO  [M:0;9fe0640122ec:35671 {}] hbase.ChoreService(370): Chore service for: master/9fe0640122ec:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-12-11T20:12:17,466 INFO  [M:0;9fe0640122ec:35671 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-11T20:12:17,467 DEBUG [M:0;9fe0640122ec:35671 {}] master.HMaster(1795): Stopping service threads
2024-12-11T20:12:17,467 INFO  [M:0;9fe0640122ec:35671 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-11T20:12:17,467 INFO  [M:0;9fe0640122ec:35671 {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-11T20:12:17,467 INFO  [M:0;9fe0640122ec:35671 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-11T20:12:17,467 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-11T20:12:17,468 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35671-0x100caf66b7a0000, quorum=127.0.0.1:58160, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-11T20:12:17,468 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35671-0x100caf66b7a0000, quorum=127.0.0.1:58160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:12:17,468 DEBUG [M:0;9fe0640122ec:35671 {}] zookeeper.ZKUtil(347): master:35671-0x100caf66b7a0000, quorum=127.0.0.1:58160, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-11T20:12:17,468 WARN  [M:0;9fe0640122ec:35671 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-11T20:12:17,469 INFO  [M:0;9fe0640122ec:35671 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/.lastflushedseqids
2024-12-11T20:12:17,494 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33959 is added to blk_1073741854_1030 (size=130)
2024-12-11T20:12:17,494 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32807 is added to blk_1073741854_1030 (size=130)
2024-12-11T20:12:17,496 INFO  [M:0;9fe0640122ec:35671 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-12-11T20:12:17,496 INFO  [M:0;9fe0640122ec:35671 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-11T20:12:17,496 DEBUG [M:0;9fe0640122ec:35671 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-11T20:12:17,496 INFO  [M:0;9fe0640122ec:35671 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T20:12:17,496 DEBUG [M:0;9fe0640122ec:35671 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T20:12:17,496 DEBUG [M:0;9fe0640122ec:35671 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-11T20:12:17,496 DEBUG [M:0;9fe0640122ec:35671 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T20:12:17,497 INFO  [M:0;9fe0640122ec:35671 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.04 KB heapSize=29.21 KB
2024-12-11T20:12:17,517 DEBUG [M:0;9fe0640122ec:35671 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b3169cb196924f29aa09431af0364041 is 82, key is hbase:meta,,1/info:regioninfo/1733947844320/Put/seqid=0
2024-12-11T20:12:17,525 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32807 is added to blk_1073741855_1031 (size=5672)
2024-12-11T20:12:17,525 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33959 is added to blk_1073741855_1031 (size=5672)
2024-12-11T20:12:17,526 INFO  [M:0;9fe0640122ec:35671 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b3169cb196924f29aa09431af0364041
2024-12-11T20:12:17,550 DEBUG [M:0;9fe0640122ec:35671 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7fd33fbbb4d546ba84f2349437336cc2 is 767, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733947845156/Put/seqid=0
2024-12-11T20:12:17,557 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32807 is added to blk_1073741856_1032 (size=6248)
2024-12-11T20:12:17,557 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33959 is added to blk_1073741856_1032 (size=6248)
2024-12-11T20:12:17,558 INFO  [M:0;9fe0640122ec:35671 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.43 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7fd33fbbb4d546ba84f2349437336cc2
2024-12-11T20:12:17,564 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45805-0x100caf66b7a0001, quorum=127.0.0.1:58160, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-11T20:12:17,564 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45805-0x100caf66b7a0001, quorum=127.0.0.1:58160, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-11T20:12:17,564 INFO  [RS:0;9fe0640122ec:45805 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-11T20:12:17,565 INFO  [RS:0;9fe0640122ec:45805 {}] regionserver.HRegionServer(1031): Exiting; stopping=9fe0640122ec,45805,1733947842477; zookeeper connection closed.
2024-12-11T20:12:17,565 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3e927cd4 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3e927cd4
2024-12-11T20:12:17,565 INFO  [M:0;9fe0640122ec:35671 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7fd33fbbb4d546ba84f2349437336cc2
2024-12-11T20:12:17,566 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-12-11T20:12:17,583 DEBUG [M:0;9fe0640122ec:35671 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8c7b64de050d44f1870dac4ce430d80f is 69, key is 9fe0640122ec,45805,1733947842477/rs:state/1733947843552/Put/seqid=0
2024-12-11T20:12:17,599 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33959 is added to blk_1073741857_1033 (size=5156)
2024-12-11T20:12:17,603 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32807 is added to blk_1073741857_1033 (size=5156)
2024-12-11T20:12:17,609 INFO  [M:0;9fe0640122ec:35671 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8c7b64de050d44f1870dac4ce430d80f
2024-12-11T20:12:17,640 INFO  [regionserver/9fe0640122ec:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-11T20:12:17,646 DEBUG [M:0;9fe0640122ec:35671 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/5f20bcb5dfb54db8b7130c2e0e0bf3c1 is 52, key is load_balancer_on/state:d/1733947844599/Put/seqid=0
2024-12-11T20:12:17,658 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32807 is added to blk_1073741858_1034 (size=5056)
2024-12-11T20:12:17,658 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33959 is added to blk_1073741858_1034 (size=5056)
2024-12-11T20:12:17,659 INFO  [M:0;9fe0640122ec:35671 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/5f20bcb5dfb54db8b7130c2e0e0bf3c1
2024-12-11T20:12:17,669 DEBUG [M:0;9fe0640122ec:35671 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b3169cb196924f29aa09431af0364041 as hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b3169cb196924f29aa09431af0364041
2024-12-11T20:12:17,686 INFO  [M:0;9fe0640122ec:35671 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b3169cb196924f29aa09431af0364041, entries=8, sequenceid=59, filesize=5.5 K
2024-12-11T20:12:17,689 DEBUG [M:0;9fe0640122ec:35671 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7fd33fbbb4d546ba84f2349437336cc2 as hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/7fd33fbbb4d546ba84f2349437336cc2
2024-12-11T20:12:17,698 INFO  [M:0;9fe0640122ec:35671 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7fd33fbbb4d546ba84f2349437336cc2
2024-12-11T20:12:17,698 INFO  [M:0;9fe0640122ec:35671 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/7fd33fbbb4d546ba84f2349437336cc2, entries=6, sequenceid=59, filesize=6.1 K
2024-12-11T20:12:17,699 DEBUG [M:0;9fe0640122ec:35671 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8c7b64de050d44f1870dac4ce430d80f as hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8c7b64de050d44f1870dac4ce430d80f
2024-12-11T20:12:17,708 INFO  [M:0;9fe0640122ec:35671 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8c7b64de050d44f1870dac4ce430d80f, entries=1, sequenceid=59, filesize=5.0 K
2024-12-11T20:12:17,709 DEBUG [M:0;9fe0640122ec:35671 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/5f20bcb5dfb54db8b7130c2e0e0bf3c1 as hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/5f20bcb5dfb54db8b7130c2e0e0bf3c1
2024-12-11T20:12:17,720 INFO  [M:0;9fe0640122ec:35671 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/5f20bcb5dfb54db8b7130c2e0e0bf3c1, entries=1, sequenceid=59, filesize=4.9 K
2024-12-11T20:12:17,725 INFO  [M:0;9fe0640122ec:35671 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.04 KB/23588, heapSize ~29.15 KB/29848, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 229ms, sequenceid=59, compaction requested=false
2024-12-11T20:12:17,738 INFO  [M:0;9fe0640122ec:35671 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T20:12:17,738 DEBUG [M:0;9fe0640122ec:35671 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733947937496Disabling compacts and flushes for region at 1733947937496Disabling writes for close at 1733947937496Obtaining lock to block concurrent updates at 1733947937497 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733947937497Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23588, getHeapSize=29848, getOffHeapSize=0, getCellsCount=70 at 1733947937497Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733947937499 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733947937499Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733947937516 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733947937516Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733947937533 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733947937549 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733947937549Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733947937565 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733947937582 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733947937583 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733947937622 (+39 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733947937645 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733947937645Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@23e964c3: reopening flushed file at 1733947937667 (+22 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@ad58ec7: reopening flushed file at 1733947937687 (+20 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4810acd3: reopening flushed file at 1733947937698 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c6e32eb: reopening flushed file at 1733947937708 (+10 ms)Finished flush of dataSize ~23.04 KB/23588, heapSize ~29.15 KB/29848, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 229ms, sequenceid=59, compaction requested=false at 1733947937725 (+17 ms)Writing region close event to WAL at 1733947937738 (+13 ms)Closed at 1733947937738
2024-12-11T20:12:17,741 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:12:17,741 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:12:17,741 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:12:17,741 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:12:17,742 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:12:17,752 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32807 is added to blk_1073741830_1006 (size=27985)
2024-12-11T20:12:17,752 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33959 is added to blk_1073741830_1006 (size=27985)
2024-12-11T20:12:17,754 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-11T20:12:17,754 INFO  [M:0;9fe0640122ec:35671 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-12-11T20:12:17,754 INFO  [M:0;9fe0640122ec:35671 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35671
2024-12-11T20:12:17,755 INFO  [M:0;9fe0640122ec:35671 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-11T20:12:17,858 INFO  [M:0;9fe0640122ec:35671 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-11T20:12:17,858 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35671-0x100caf66b7a0000, quorum=127.0.0.1:58160, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-11T20:12:17,858 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35671-0x100caf66b7a0000, quorum=127.0.0.1:58160, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-11T20:12:17,863 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1bf97579{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T20:12:17,866 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@22b88bcb{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-11T20:12:17,866 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-11T20:12:17,866 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2d48d695{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-11T20:12:17,867 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@11effdcd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/26824f70-ac36-0745-fb30-f5180937eb59/hadoop.log.dir/,STOPPED}
2024-12-11T20:12:17,871 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-11T20:12:17,871 WARN  [BP-560241680-172.17.0.2-1733947838484 heartbeating to localhost/127.0.0.1:36219 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-11T20:12:17,871 WARN  [BP-560241680-172.17.0.2-1733947838484 heartbeating to localhost/127.0.0.1:36219 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-560241680-172.17.0.2-1733947838484 (Datanode Uuid e1bb997a-d108-4896-aa3f-f6c561776e0d) service to localhost/127.0.0.1:36219
2024-12-11T20:12:17,871 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-11T20:12:17,872 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/26824f70-ac36-0745-fb30-f5180937eb59/cluster_5e5800cd-2c49-c72c-329f-be0e65d0b9ec/data/data3/current/BP-560241680-172.17.0.2-1733947838484 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T20:12:17,872 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/26824f70-ac36-0745-fb30-f5180937eb59/cluster_5e5800cd-2c49-c72c-329f-be0e65d0b9ec/data/data4/current/BP-560241680-172.17.0.2-1733947838484 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T20:12:17,873 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-11T20:12:17,877 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7b07d1ba{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T20:12:17,878 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@43e0a762{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-11T20:12:17,878 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-11T20:12:17,878 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@371e191c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-11T20:12:17,878 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@28778f0f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/26824f70-ac36-0745-fb30-f5180937eb59/hadoop.log.dir/,STOPPED}
2024-12-11T20:12:17,881 WARN  [BP-560241680-172.17.0.2-1733947838484 heartbeating to localhost/127.0.0.1:36219 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-11T20:12:17,881 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-11T20:12:17,881 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-11T20:12:17,881 WARN  [BP-560241680-172.17.0.2-1733947838484 heartbeating to localhost/127.0.0.1:36219 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-560241680-172.17.0.2-1733947838484 (Datanode Uuid 3215035f-f1b0-4122-b293-4be79df83ad6) service to localhost/127.0.0.1:36219
2024-12-11T20:12:17,882 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/26824f70-ac36-0745-fb30-f5180937eb59/cluster_5e5800cd-2c49-c72c-329f-be0e65d0b9ec/data/data1/current/BP-560241680-172.17.0.2-1733947838484 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T20:12:17,882 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/26824f70-ac36-0745-fb30-f5180937eb59/cluster_5e5800cd-2c49-c72c-329f-be0e65d0b9ec/data/data2/current/BP-560241680-172.17.0.2-1733947838484 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T20:12:17,883 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-11T20:12:17,895 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@735fa16a{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-11T20:12:17,896 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6c26a5a3{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-11T20:12:17,896 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-11T20:12:17,897 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@70be1389{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-11T20:12:17,897 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@ddc8467{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/26824f70-ac36-0745-fb30-f5180937eb59/hadoop.log.dir/,STOPPED}
2024-12-11T20:12:17,906 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-12-11T20:12:17,939 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-12-11T20:12:17,950 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=80 (was 12)
Potentially hanging thread: RPCClient-NioEventLoopGroup-4-1
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:36219 from jenkins.hfs.0
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: sync.0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: Timer for 'HBase' metrics system
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: sync.3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: HBase-Metrics2-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:36219 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: Time-limited test.named-queue-events-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47)
	app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56)
	app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159)
	app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-2-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
	app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-4-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-2-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: SessionTracker
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163)

Potentially hanging thread: sync.2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-3-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: regionserver/9fe0640122ec:0.procedureResultReporter
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75)

Potentially hanging thread: region-location-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-5-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: sync.4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-4-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-5-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Async-Client-Retry-Timer-pool-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: GcTimeMonitor obsWindow = 60000, sleepInterval = 5000, maxGcTimePerc = 100
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.util.GcTimeMonitor.run(GcTimeMonitor.java:161)

Potentially hanging thread: org.apache.hadoop.hdfs.PeerCache@18e1792d
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253)
	app//org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46)
	app//org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36219
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Monitor thread for TaskMonitor
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-2-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-1-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: master/9fe0640122ec:0:becomeActiveMaster-MemStoreChunkPool Statistics
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:36219
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-5-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: SnapshotHandlerChoreCleaner
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-3-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36219
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.0@localhost:36219
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: SSL Certificates Store Monitor
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.Object.wait(Object.java:338)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:537)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: weak-ref-cleaner-strictcontextstorage
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
	app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-1-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: sync.3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36219
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:36219 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: ForkJoinPool-2-worker-3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: nioEventLoopGroup-4-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: ForkJoinPool-2-worker-2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: nioEventLoopGroup-3-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: master/9fe0640122ec:0:becomeActiveMaster-MemStoreChunkPool Statistics
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: sync.1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: HMaster-EventLoopGroup-1-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RpcClient-timer-pool-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=402 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=167 (was 170), ProcessCount=11 (was 11), AvailableMemoryMB=940 (was 1296)
2024-12-11T20:12:17,958 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=81, OpenFileDescriptor=402, MaxFileDescriptor=1048576, SystemLoadAverage=167, ProcessCount=11, AvailableMemoryMB=940
2024-12-11T20:12:17,958 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-11T20:12:17,959 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/26824f70-ac36-0745-fb30-f5180937eb59/hadoop.log.dir so I do NOT create it in target/test-data/2a4de8f6-5a88-0e0e-aab1-1c7fba13c249
2024-12-11T20:12:17,959 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/26824f70-ac36-0745-fb30-f5180937eb59/hadoop.tmp.dir so I do NOT create it in target/test-data/2a4de8f6-5a88-0e0e-aab1-1c7fba13c249
2024-12-11T20:12:17,959 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2a4de8f6-5a88-0e0e-aab1-1c7fba13c249/cluster_a95c7252-f29d-d111-d7d7-455ec09db6de, deleteOnExit=true
2024-12-11T20:12:17,959 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-12-11T20:12:17,959 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2a4de8f6-5a88-0e0e-aab1-1c7fba13c249/test.cache.data in system properties and HBase conf
2024-12-11T20:12:17,959 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2a4de8f6-5a88-0e0e-aab1-1c7fba13c249/hadoop.tmp.dir in system properties and HBase conf
2024-12-11T20:12:17,959 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2a4de8f6-5a88-0e0e-aab1-1c7fba13c249/hadoop.log.dir in system properties and HBase conf
2024-12-11T20:12:17,959 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2a4de8f6-5a88-0e0e-aab1-1c7fba13c249/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-11T20:12:17,960 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2a4de8f6-5a88-0e0e-aab1-1c7fba13c249/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-11T20:12:17,960 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-12-11T20:12:17,960 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-11T20:12:17,960 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2a4de8f6-5a88-0e0e-aab1-1c7fba13c249/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-11T20:12:17,960 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2a4de8f6-5a88-0e0e-aab1-1c7fba13c249/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-11T20:12:17,960 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2a4de8f6-5a88-0e0e-aab1-1c7fba13c249/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-11T20:12:17,960 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2a4de8f6-5a88-0e0e-aab1-1c7fba13c249/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-11T20:12:17,960 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2a4de8f6-5a88-0e0e-aab1-1c7fba13c249/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-11T20:12:17,960 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2a4de8f6-5a88-0e0e-aab1-1c7fba13c249/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-11T20:12:17,960 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2a4de8f6-5a88-0e0e-aab1-1c7fba13c249/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-11T20:12:17,961 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2a4de8f6-5a88-0e0e-aab1-1c7fba13c249/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-11T20:12:17,961 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2a4de8f6-5a88-0e0e-aab1-1c7fba13c249/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-11T20:12:17,961 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2a4de8f6-5a88-0e0e-aab1-1c7fba13c249/nfs.dump.dir in system properties and HBase conf
2024-12-11T20:12:17,961 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2a4de8f6-5a88-0e0e-aab1-1c7fba13c249/java.io.tmpdir in system properties and HBase conf
2024-12-11T20:12:17,961 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2a4de8f6-5a88-0e0e-aab1-1c7fba13c249/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-11T20:12:17,961 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2a4de8f6-5a88-0e0e-aab1-1c7fba13c249/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-11T20:12:17,961 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2a4de8f6-5a88-0e0e-aab1-1c7fba13c249/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-11T20:12:17,975 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-11T20:12:18,057 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T20:12:18,065 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-11T20:12:18,066 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-11T20:12:18,066 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-11T20:12:18,066 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-11T20:12:18,067 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T20:12:18,068 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6c443180{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2a4de8f6-5a88-0e0e-aab1-1c7fba13c249/hadoop.log.dir/,AVAILABLE}
2024-12-11T20:12:18,069 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@10885b70{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-11T20:12:18,197 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@245f97f3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2a4de8f6-5a88-0e0e-aab1-1c7fba13c249/java.io.tmpdir/jetty-localhost-38945-hadoop-hdfs-3_4_1-tests_jar-_-any-16609729910272735737/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-11T20:12:18,198 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@219d080{HTTP/1.1, (http/1.1)}{localhost:38945}
2024-12-11T20:12:18,198 INFO  [Time-limited test {}] server.Server(415): Started @101794ms
2024-12-11T20:12:18,212 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-11T20:12:18,281 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T20:12:18,284 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-11T20:12:18,285 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-11T20:12:18,285 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-11T20:12:18,285 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-11T20:12:18,286 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@b5aedfa{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2a4de8f6-5a88-0e0e-aab1-1c7fba13c249/hadoop.log.dir/,AVAILABLE}
2024-12-11T20:12:18,286 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1d281952{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-11T20:12:18,403 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7bd218c7{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2a4de8f6-5a88-0e0e-aab1-1c7fba13c249/java.io.tmpdir/jetty-localhost-32917-hadoop-hdfs-3_4_1-tests_jar-_-any-9195843253747158282/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T20:12:18,403 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5a9b9566{HTTP/1.1, (http/1.1)}{localhost:32917}
2024-12-11T20:12:18,403 INFO  [Time-limited test {}] server.Server(415): Started @101999ms
2024-12-11T20:12:18,405 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-11T20:12:18,445 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T20:12:18,450 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-11T20:12:18,453 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-11T20:12:18,453 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-11T20:12:18,453 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-11T20:12:18,457 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@30ebe7e3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2a4de8f6-5a88-0e0e-aab1-1c7fba13c249/hadoop.log.dir/,AVAILABLE}
2024-12-11T20:12:18,457 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@254e0164{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-11T20:12:18,500 WARN  [Thread-440 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2a4de8f6-5a88-0e0e-aab1-1c7fba13c249/cluster_a95c7252-f29d-d111-d7d7-455ec09db6de/data/data1/current/BP-884266491-172.17.0.2-1733947937993/current, will proceed with Du for space computation calculation, 
2024-12-11T20:12:18,500 WARN  [Thread-441 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2a4de8f6-5a88-0e0e-aab1-1c7fba13c249/cluster_a95c7252-f29d-d111-d7d7-455ec09db6de/data/data2/current/BP-884266491-172.17.0.2-1733947937993/current, will proceed with Du for space computation calculation, 
2024-12-11T20:12:18,528 WARN  [Thread-419 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-11T20:12:18,532 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6440a68a2c3f862e with lease ID 0xffdcf1fcfe9ba14a: Processing first storage report for DS-360fc53a-932d-47e2-94e1-efccbbe1a932 from datanode DatanodeRegistration(127.0.0.1:39723, datanodeUuid=e2a77dfe-2623-4901-975a-c3c2425f2e9e, infoPort=36175, infoSecurePort=0, ipcPort=43371, storageInfo=lv=-57;cid=testClusterID;nsid=2120719304;c=1733947937993)
2024-12-11T20:12:18,532 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6440a68a2c3f862e with lease ID 0xffdcf1fcfe9ba14a: from storage DS-360fc53a-932d-47e2-94e1-efccbbe1a932 node DatanodeRegistration(127.0.0.1:39723, datanodeUuid=e2a77dfe-2623-4901-975a-c3c2425f2e9e, infoPort=36175, infoSecurePort=0, ipcPort=43371, storageInfo=lv=-57;cid=testClusterID;nsid=2120719304;c=1733947937993), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T20:12:18,532 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6440a68a2c3f862e with lease ID 0xffdcf1fcfe9ba14a: Processing first storage report for DS-83841e67-f8b1-4ae2-9189-e235e5692e46 from datanode DatanodeRegistration(127.0.0.1:39723, datanodeUuid=e2a77dfe-2623-4901-975a-c3c2425f2e9e, infoPort=36175, infoSecurePort=0, ipcPort=43371, storageInfo=lv=-57;cid=testClusterID;nsid=2120719304;c=1733947937993)
2024-12-11T20:12:18,532 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6440a68a2c3f862e with lease ID 0xffdcf1fcfe9ba14a: from storage DS-83841e67-f8b1-4ae2-9189-e235e5692e46 node DatanodeRegistration(127.0.0.1:39723, datanodeUuid=e2a77dfe-2623-4901-975a-c3c2425f2e9e, infoPort=36175, infoSecurePort=0, ipcPort=43371, storageInfo=lv=-57;cid=testClusterID;nsid=2120719304;c=1733947937993), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T20:12:18,588 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@156b26b5{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2a4de8f6-5a88-0e0e-aab1-1c7fba13c249/java.io.tmpdir/jetty-localhost-42737-hadoop-hdfs-3_4_1-tests_jar-_-any-156812623339191177/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T20:12:18,588 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3dd99d02{HTTP/1.1, (http/1.1)}{localhost:42737}
2024-12-11T20:12:18,588 INFO  [Time-limited test {}] server.Server(415): Started @102184ms
2024-12-11T20:12:18,590 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-11T20:12:18,685 WARN  [Thread-466 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2a4de8f6-5a88-0e0e-aab1-1c7fba13c249/cluster_a95c7252-f29d-d111-d7d7-455ec09db6de/data/data3/current/BP-884266491-172.17.0.2-1733947937993/current, will proceed with Du for space computation calculation, 
2024-12-11T20:12:18,685 WARN  [Thread-467 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2a4de8f6-5a88-0e0e-aab1-1c7fba13c249/cluster_a95c7252-f29d-d111-d7d7-455ec09db6de/data/data4/current/BP-884266491-172.17.0.2-1733947937993/current, will proceed with Du for space computation calculation, 
2024-12-11T20:12:18,712 WARN  [Thread-455 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-11T20:12:18,715 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x41d8717cc6eb71f2 with lease ID 0xffdcf1fcfe9ba14b: Processing first storage report for DS-e9d1b52f-2766-44a9-9050-89ea2e669088 from datanode DatanodeRegistration(127.0.0.1:33365, datanodeUuid=37739307-4403-4599-8793-3e4474d318eb, infoPort=41319, infoSecurePort=0, ipcPort=39999, storageInfo=lv=-57;cid=testClusterID;nsid=2120719304;c=1733947937993)
2024-12-11T20:12:18,715 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x41d8717cc6eb71f2 with lease ID 0xffdcf1fcfe9ba14b: from storage DS-e9d1b52f-2766-44a9-9050-89ea2e669088 node DatanodeRegistration(127.0.0.1:33365, datanodeUuid=37739307-4403-4599-8793-3e4474d318eb, infoPort=41319, infoSecurePort=0, ipcPort=39999, storageInfo=lv=-57;cid=testClusterID;nsid=2120719304;c=1733947937993), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T20:12:18,716 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x41d8717cc6eb71f2 with lease ID 0xffdcf1fcfe9ba14b: Processing first storage report for DS-318da236-0468-4ed9-815e-5cf4018dde60 from datanode DatanodeRegistration(127.0.0.1:33365, datanodeUuid=37739307-4403-4599-8793-3e4474d318eb, infoPort=41319, infoSecurePort=0, ipcPort=39999, storageInfo=lv=-57;cid=testClusterID;nsid=2120719304;c=1733947937993)
2024-12-11T20:12:18,716 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x41d8717cc6eb71f2 with lease ID 0xffdcf1fcfe9ba14b: from storage DS-318da236-0468-4ed9-815e-5cf4018dde60 node DatanodeRegistration(127.0.0.1:33365, datanodeUuid=37739307-4403-4599-8793-3e4474d318eb, infoPort=41319, infoSecurePort=0, ipcPort=39999, storageInfo=lv=-57;cid=testClusterID;nsid=2120719304;c=1733947937993), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T20:12:18,723 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2a4de8f6-5a88-0e0e-aab1-1c7fba13c249
2024-12-11T20:12:18,732 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2a4de8f6-5a88-0e0e-aab1-1c7fba13c249/cluster_a95c7252-f29d-d111-d7d7-455ec09db6de/zookeeper_0, clientPort=57481, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2a4de8f6-5a88-0e0e-aab1-1c7fba13c249/cluster_a95c7252-f29d-d111-d7d7-455ec09db6de/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2a4de8f6-5a88-0e0e-aab1-1c7fba13c249/cluster_a95c7252-f29d-d111-d7d7-455ec09db6de/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-11T20:12:18,733 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=57481
2024-12-11T20:12:18,733 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T20:12:18,736 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T20:12:18,755 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39723 is added to blk_1073741825_1001 (size=7)
2024-12-11T20:12:18,756 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33365 is added to blk_1073741825_1001 (size=7)
2024-12-11T20:12:18,759 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44595/user/jenkins/test-data/8e0469c0-f728-e9f1-3444-f2c54a669985 with version=8
2024-12-11T20:12:18,759 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/hbase-staging
2024-12-11T20:12:18,762 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/9fe0640122ec:0 server-side Connection retries=45
2024-12-11T20:12:18,762 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-11T20:12:18,762 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-11T20:12:18,762 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-11T20:12:18,762 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-11T20:12:18,762 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-11T20:12:18,762 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-12-11T20:12:18,763 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-11T20:12:18,763 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45133
2024-12-11T20:12:18,765 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:45133 connecting to ZooKeeper ensemble=127.0.0.1:57481
2024-12-11T20:12:18,772 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:451330x0, quorum=127.0.0.1:57481, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-11T20:12:18,773 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:45133-0x100caf7ea240000 connected
2024-12-11T20:12:18,792 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T20:12:18,795 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T20:12:18,798 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45133-0x100caf7ea240000, quorum=127.0.0.1:57481, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-11T20:12:18,799 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44595/user/jenkins/test-data/8e0469c0-f728-e9f1-3444-f2c54a669985, hbase.cluster.distributed=false
2024-12-11T20:12:18,801 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45133-0x100caf7ea240000, quorum=127.0.0.1:57481, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-11T20:12:18,805 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45133
2024-12-11T20:12:18,805 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45133
2024-12-11T20:12:18,805 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45133
2024-12-11T20:12:18,806 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45133
2024-12-11T20:12:18,806 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45133
2024-12-11T20:12:18,826 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/9fe0640122ec:0 server-side Connection retries=45
2024-12-11T20:12:18,826 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-11T20:12:18,826 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-11T20:12:18,826 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-11T20:12:18,826 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-11T20:12:18,826 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-11T20:12:18,826 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-11T20:12:18,827 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-11T20:12:18,827 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33749
2024-12-11T20:12:18,829 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33749 connecting to ZooKeeper ensemble=127.0.0.1:57481
2024-12-11T20:12:18,829 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T20:12:18,832 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T20:12:18,837 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:337490x0, quorum=127.0.0.1:57481, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-11T20:12:18,838 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33749-0x100caf7ea240001, quorum=127.0.0.1:57481, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-11T20:12:18,838 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33749-0x100caf7ea240001 connected
2024-12-11T20:12:18,838 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-11T20:12:18,839 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-11T20:12:18,840 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33749-0x100caf7ea240001, quorum=127.0.0.1:57481, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-11T20:12:18,841 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33749-0x100caf7ea240001, quorum=127.0.0.1:57481, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-11T20:12:18,842 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33749
2024-12-11T20:12:18,842 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33749
2024-12-11T20:12:18,844 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33749
2024-12-11T20:12:18,847 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33749
2024-12-11T20:12:18,851 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33749
2024-12-11T20:12:18,866 DEBUG [M:0;9fe0640122ec:45133 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;9fe0640122ec:45133
2024-12-11T20:12:18,866 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/9fe0640122ec,45133,1733947938761
2024-12-11T20:12:18,868 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45133-0x100caf7ea240000, quorum=127.0.0.1:57481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-11T20:12:18,869 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33749-0x100caf7ea240001, quorum=127.0.0.1:57481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-11T20:12:18,869 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45133-0x100caf7ea240000, quorum=127.0.0.1:57481, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/9fe0640122ec,45133,1733947938761
2024-12-11T20:12:18,871 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33749-0x100caf7ea240001, quorum=127.0.0.1:57481, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-11T20:12:18,871 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45133-0x100caf7ea240000, quorum=127.0.0.1:57481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:12:18,871 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33749-0x100caf7ea240001, quorum=127.0.0.1:57481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:12:18,873 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45133-0x100caf7ea240000, quorum=127.0.0.1:57481, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-11T20:12:18,873 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/9fe0640122ec,45133,1733947938761 from backup master directory
2024-12-11T20:12:18,875 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33749-0x100caf7ea240001, quorum=127.0.0.1:57481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-11T20:12:18,875 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45133-0x100caf7ea240000, quorum=127.0.0.1:57481, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/9fe0640122ec,45133,1733947938761
2024-12-11T20:12:18,875 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45133-0x100caf7ea240000, quorum=127.0.0.1:57481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-11T20:12:18,875 WARN  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-11T20:12:18,875 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=9fe0640122ec,45133,1733947938761
2024-12-11T20:12:18,888 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44595/user/jenkins/test-data/8e0469c0-f728-e9f1-3444-f2c54a669985/hbase.id] with ID: c12fb9b2-934e-4e25-b8f8-8881cc86de0c
2024-12-11T20:12:18,888 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44595/user/jenkins/test-data/8e0469c0-f728-e9f1-3444-f2c54a669985/.tmp/hbase.id
2024-12-11T20:12:18,897 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33365 is added to blk_1073741826_1002 (size=42)
2024-12-11T20:12:18,898 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39723 is added to blk_1073741826_1002 (size=42)
2024-12-11T20:12:18,898 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44595/user/jenkins/test-data/8e0469c0-f728-e9f1-3444-f2c54a669985/.tmp/hbase.id]:[hdfs://localhost:44595/user/jenkins/test-data/8e0469c0-f728-e9f1-3444-f2c54a669985/hbase.id]
2024-12-11T20:12:18,913 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T20:12:18,913 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-12-11T20:12:18,915 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms.
2024-12-11T20:12:18,918 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33749-0x100caf7ea240001, quorum=127.0.0.1:57481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:12:18,918 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45133-0x100caf7ea240000, quorum=127.0.0.1:57481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:12:18,929 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39723 is added to blk_1073741827_1003 (size=196)
2024-12-11T20:12:18,929 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33365 is added to blk_1073741827_1003 (size=196)
2024-12-11T20:12:18,931 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-11T20:12:18,932 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-11T20:12:18,932 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-11T20:12:18,946 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33365 is added to blk_1073741828_1004 (size=1189)
2024-12-11T20:12:18,946 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39723 is added to blk_1073741828_1004 (size=1189)
2024-12-11T20:12:18,947 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44595/user/jenkins/test-data/8e0469c0-f728-e9f1-3444-f2c54a669985/MasterData/data/master/store
2024-12-11T20:12:18,955 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39723 is added to blk_1073741829_1005 (size=34)
2024-12-11T20:12:18,956 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33365 is added to blk_1073741829_1005 (size=34)
2024-12-11T20:12:18,958 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T20:12:18,959 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-11T20:12:18,959 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T20:12:18,959 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T20:12:18,959 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-11T20:12:18,959 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T20:12:18,959 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T20:12:18,959 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733947938958Disabling compacts and flushes for region at 1733947938958Disabling writes for close at 1733947938959 (+1 ms)Writing region close event to WAL at 1733947938959Closed at 1733947938959
2024-12-11T20:12:18,961 WARN  [master/9fe0640122ec:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44595/user/jenkins/test-data/8e0469c0-f728-e9f1-3444-f2c54a669985/MasterData/data/master/store/.initializing
2024-12-11T20:12:18,961 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44595/user/jenkins/test-data/8e0469c0-f728-e9f1-3444-f2c54a669985/MasterData/WALs/9fe0640122ec,45133,1733947938761
2024-12-11T20:12:18,966 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9fe0640122ec%2C45133%2C1733947938761, suffix=, logDir=hdfs://localhost:44595/user/jenkins/test-data/8e0469c0-f728-e9f1-3444-f2c54a669985/MasterData/WALs/9fe0640122ec,45133,1733947938761, archiveDir=hdfs://localhost:44595/user/jenkins/test-data/8e0469c0-f728-e9f1-3444-f2c54a669985/MasterData/oldWALs, maxLogs=10
2024-12-11T20:12:18,967 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9fe0640122ec%2C45133%2C1733947938761.1733947938967
2024-12-11T20:12:18,973 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/8e0469c0-f728-e9f1-3444-f2c54a669985/MasterData/WALs/9fe0640122ec,45133,1733947938761/9fe0640122ec%2C45133%2C1733947938761.1733947938967
2024-12-11T20:12:18,974 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36175:36175),(127.0.0.1/127.0.0.1:41319:41319)]
2024-12-11T20:12:18,975 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-11T20:12:18,975 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T20:12:18,975 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T20:12:18,975 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T20:12:18,977 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-11T20:12:18,979 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-11T20:12:18,979 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:12:18,980 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T20:12:18,980 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-11T20:12:18,982 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-11T20:12:18,982 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:12:18,982 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-11T20:12:18,983 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-11T20:12:18,985 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-11T20:12:18,985 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:12:18,986 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-11T20:12:18,986 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-11T20:12:18,987 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-11T20:12:18,987 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:12:18,988 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-11T20:12:18,988 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T20:12:18,989 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44595/user/jenkins/test-data/8e0469c0-f728-e9f1-3444-f2c54a669985/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-11T20:12:18,990 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44595/user/jenkins/test-data/8e0469c0-f728-e9f1-3444-f2c54a669985/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-11T20:12:18,992 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T20:12:18,992 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T20:12:18,993 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-11T20:12:18,996 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T20:12:18,999 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44595/user/jenkins/test-data/8e0469c0-f728-e9f1-3444-f2c54a669985/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-11T20:12:18,999 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=780646, jitterRate=-0.0073585063219070435}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-11T20:12:19,001 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1733947938976Initializing all the Stores at 1733947938977 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733947938977Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733947938977Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733947938977Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733947938977Cleaning up temporary data from old regions at 1733947938992 (+15 ms)Region opened successfully at 1733947939001 (+9 ms)
2024-12-11T20:12:19,002 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-11T20:12:19,006 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42db1488, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9fe0640122ec/172.17.0.2:0
2024-12-11T20:12:19,007 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-12-11T20:12:19,008 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-11T20:12:19,008 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-11T20:12:19,008 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-11T20:12:19,008 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-12-11T20:12:19,009 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-12-11T20:12:19,009 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-11T20:12:19,012 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-11T20:12:19,013 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45133-0x100caf7ea240000, quorum=127.0.0.1:57481, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-11T20:12:19,014 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-12-11T20:12:19,015 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-11T20:12:19,016 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45133-0x100caf7ea240000, quorum=127.0.0.1:57481, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-11T20:12:19,017 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-12-11T20:12:19,018 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-11T20:12:19,019 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45133-0x100caf7ea240000, quorum=127.0.0.1:57481, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-11T20:12:19,020 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-12-11T20:12:19,021 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45133-0x100caf7ea240000, quorum=127.0.0.1:57481, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-11T20:12:19,022 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-12-11T20:12:19,024 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45133-0x100caf7ea240000, quorum=127.0.0.1:57481, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-11T20:12:19,026 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-11T20:12:19,028 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33749-0x100caf7ea240001, quorum=127.0.0.1:57481, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-11T20:12:19,028 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45133-0x100caf7ea240000, quorum=127.0.0.1:57481, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-11T20:12:19,028 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33749-0x100caf7ea240001, quorum=127.0.0.1:57481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:12:19,028 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45133-0x100caf7ea240000, quorum=127.0.0.1:57481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:12:19,028 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=9fe0640122ec,45133,1733947938761, sessionid=0x100caf7ea240000, setting cluster-up flag (Was=false)
2024-12-11T20:12:19,032 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45133-0x100caf7ea240000, quorum=127.0.0.1:57481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:12:19,032 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33749-0x100caf7ea240001, quorum=127.0.0.1:57481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:12:19,038 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-11T20:12:19,039 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9fe0640122ec,45133,1733947938761
2024-12-11T20:12:19,043 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33749-0x100caf7ea240001, quorum=127.0.0.1:57481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:12:19,043 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45133-0x100caf7ea240000, quorum=127.0.0.1:57481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:12:19,048 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-11T20:12:19,049 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9fe0640122ec,45133,1733947938761
2024-12-11T20:12:19,053 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44595/user/jenkins/test-data/8e0469c0-f728-e9f1-3444-f2c54a669985/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-12-11T20:12:19,055 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-12-11T20:12:19,055 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-12-11T20:12:19,055 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-11T20:12:19,055 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 9fe0640122ec,45133,1733947938761
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-11T20:12:19,057 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/9fe0640122ec:0, corePoolSize=5, maxPoolSize=5
2024-12-11T20:12:19,057 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/9fe0640122ec:0, corePoolSize=5, maxPoolSize=5
2024-12-11T20:12:19,057 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/9fe0640122ec:0, corePoolSize=5, maxPoolSize=5
2024-12-11T20:12:19,057 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/9fe0640122ec:0, corePoolSize=5, maxPoolSize=5
2024-12-11T20:12:19,057 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/9fe0640122ec:0, corePoolSize=10, maxPoolSize=10
2024-12-11T20:12:19,057 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:12:19,057 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/9fe0640122ec:0, corePoolSize=2, maxPoolSize=2
2024-12-11T20:12:19,058 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:12:19,062 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-11T20:12:19,062 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-12-11T20:12:19,064 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:12:19,064 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-11T20:12:19,076 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733947969076
2024-12-11T20:12:19,076 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-11T20:12:19,076 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-11T20:12:19,077 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-11T20:12:19,077 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-11T20:12:19,077 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-11T20:12:19,077 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-11T20:12:19,080 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-11T20:12:19,081 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-11T20:12:19,081 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-11T20:12:19,081 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-11T20:12:19,084 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-11T20:12:19,085 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-11T20:12:19,085 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/9fe0640122ec:0:becomeActiveMaster-HFileCleaner.large.0-1733947939085,5,FailOnTimeoutGroup]
2024-12-11T20:12:19,088 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39723 is added to blk_1073741831_1007 (size=1321)
2024-12-11T20:12:19,088 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33365 is added to blk_1073741831_1007 (size=1321)
2024-12-11T20:12:19,088 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/9fe0640122ec:0:becomeActiveMaster-HFileCleaner.small.0-1733947939085,5,FailOnTimeoutGroup]
2024-12-11T20:12:19,088 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-11T20:12:19,089 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-11T20:12:19,089 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-11T20:12:19,089 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-11T20:12:19,154 INFO  [RS:0;9fe0640122ec:33749 {}] regionserver.HRegionServer(746): ClusterId : c12fb9b2-934e-4e25-b8f8-8881cc86de0c
2024-12-11T20:12:19,154 DEBUG [RS:0;9fe0640122ec:33749 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-11T20:12:19,157 DEBUG [RS:0;9fe0640122ec:33749 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-11T20:12:19,157 DEBUG [RS:0;9fe0640122ec:33749 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-11T20:12:19,160 DEBUG [RS:0;9fe0640122ec:33749 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-11T20:12:19,160 DEBUG [RS:0;9fe0640122ec:33749 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f737a46, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9fe0640122ec/172.17.0.2:0
2024-12-11T20:12:19,174 DEBUG [RS:0;9fe0640122ec:33749 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;9fe0640122ec:33749
2024-12-11T20:12:19,174 INFO  [RS:0;9fe0640122ec:33749 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-11T20:12:19,174 INFO  [RS:0;9fe0640122ec:33749 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-11T20:12:19,174 DEBUG [RS:0;9fe0640122ec:33749 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-11T20:12:19,175 INFO  [RS:0;9fe0640122ec:33749 {}] regionserver.HRegionServer(2659): reportForDuty to master=9fe0640122ec,45133,1733947938761 with port=33749, startcode=1733947938826
2024-12-11T20:12:19,175 DEBUG [RS:0;9fe0640122ec:33749 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-11T20:12:19,179 INFO  [HMaster-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33989, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService
2024-12-11T20:12:19,180 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45133 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9fe0640122ec,33749,1733947938826
2024-12-11T20:12:19,180 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45133 {}] master.ServerManager(517): Registering regionserver=9fe0640122ec,33749,1733947938826
2024-12-11T20:12:19,183 DEBUG [RS:0;9fe0640122ec:33749 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44595/user/jenkins/test-data/8e0469c0-f728-e9f1-3444-f2c54a669985
2024-12-11T20:12:19,183 DEBUG [RS:0;9fe0640122ec:33749 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44595
2024-12-11T20:12:19,183 DEBUG [RS:0;9fe0640122ec:33749 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-11T20:12:19,185 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45133-0x100caf7ea240000, quorum=127.0.0.1:57481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-11T20:12:19,186 DEBUG [RS:0;9fe0640122ec:33749 {}] zookeeper.ZKUtil(111): regionserver:33749-0x100caf7ea240001, quorum=127.0.0.1:57481, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9fe0640122ec,33749,1733947938826
2024-12-11T20:12:19,186 WARN  [RS:0;9fe0640122ec:33749 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-11T20:12:19,187 INFO  [RS:0;9fe0640122ec:33749 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-11T20:12:19,187 DEBUG [RS:0;9fe0640122ec:33749 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44595/user/jenkins/test-data/8e0469c0-f728-e9f1-3444-f2c54a669985/WALs/9fe0640122ec,33749,1733947938826
2024-12-11T20:12:19,189 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9fe0640122ec,33749,1733947938826]
2024-12-11T20:12:19,193 INFO  [RS:0;9fe0640122ec:33749 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-11T20:12:19,200 INFO  [RS:0;9fe0640122ec:33749 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-11T20:12:19,203 INFO  [RS:0;9fe0640122ec:33749 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-11T20:12:19,204 INFO  [RS:0;9fe0640122ec:33749 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-11T20:12:19,208 INFO  [RS:0;9fe0640122ec:33749 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-11T20:12:19,213 INFO  [RS:0;9fe0640122ec:33749 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-11T20:12:19,213 INFO  [RS:0;9fe0640122ec:33749 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-11T20:12:19,213 DEBUG [RS:0;9fe0640122ec:33749 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:12:19,213 DEBUG [RS:0;9fe0640122ec:33749 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:12:19,214 DEBUG [RS:0;9fe0640122ec:33749 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:12:19,214 DEBUG [RS:0;9fe0640122ec:33749 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:12:19,214 DEBUG [RS:0;9fe0640122ec:33749 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:12:19,214 DEBUG [RS:0;9fe0640122ec:33749 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9fe0640122ec:0, corePoolSize=2, maxPoolSize=2
2024-12-11T20:12:19,214 DEBUG [RS:0;9fe0640122ec:33749 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:12:19,214 DEBUG [RS:0;9fe0640122ec:33749 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:12:19,214 DEBUG [RS:0;9fe0640122ec:33749 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:12:19,214 DEBUG [RS:0;9fe0640122ec:33749 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:12:19,214 DEBUG [RS:0;9fe0640122ec:33749 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:12:19,214 DEBUG [RS:0;9fe0640122ec:33749 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:12:19,215 DEBUG [RS:0;9fe0640122ec:33749 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9fe0640122ec:0, corePoolSize=3, maxPoolSize=3
2024-12-11T20:12:19,215 DEBUG [RS:0;9fe0640122ec:33749 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9fe0640122ec:0, corePoolSize=3, maxPoolSize=3
2024-12-11T20:12:19,221 INFO  [RS:0;9fe0640122ec:33749 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-11T20:12:19,221 INFO  [RS:0;9fe0640122ec:33749 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-11T20:12:19,221 INFO  [RS:0;9fe0640122ec:33749 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-11T20:12:19,221 INFO  [RS:0;9fe0640122ec:33749 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-11T20:12:19,221 INFO  [RS:0;9fe0640122ec:33749 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-11T20:12:19,221 INFO  [RS:0;9fe0640122ec:33749 {}] hbase.ChoreService(168): Chore ScheduledChore name=9fe0640122ec,33749,1733947938826-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-11T20:12:19,238 INFO  [RS:0;9fe0640122ec:33749 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-11T20:12:19,239 INFO  [RS:0;9fe0640122ec:33749 {}] hbase.ChoreService(168): Chore ScheduledChore name=9fe0640122ec,33749,1733947938826-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-11T20:12:19,239 INFO  [RS:0;9fe0640122ec:33749 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T20:12:19,239 INFO  [RS:0;9fe0640122ec:33749 {}] regionserver.Replication(171): 9fe0640122ec,33749,1733947938826 started
2024-12-11T20:12:19,256 INFO  [RS:0;9fe0640122ec:33749 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T20:12:19,256 INFO  [RS:0;9fe0640122ec:33749 {}] regionserver.HRegionServer(1482): Serving as 9fe0640122ec,33749,1733947938826, RpcServer on 9fe0640122ec/172.17.0.2:33749, sessionid=0x100caf7ea240001
2024-12-11T20:12:19,256 DEBUG [RS:0;9fe0640122ec:33749 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-11T20:12:19,257 DEBUG [RS:0;9fe0640122ec:33749 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9fe0640122ec,33749,1733947938826
2024-12-11T20:12:19,257 DEBUG [RS:0;9fe0640122ec:33749 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9fe0640122ec,33749,1733947938826'
2024-12-11T20:12:19,257 DEBUG [RS:0;9fe0640122ec:33749 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-11T20:12:19,257 DEBUG [RS:0;9fe0640122ec:33749 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-11T20:12:19,258 DEBUG [RS:0;9fe0640122ec:33749 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-11T20:12:19,258 DEBUG [RS:0;9fe0640122ec:33749 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-11T20:12:19,258 DEBUG [RS:0;9fe0640122ec:33749 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9fe0640122ec,33749,1733947938826
2024-12-11T20:12:19,258 DEBUG [RS:0;9fe0640122ec:33749 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9fe0640122ec,33749,1733947938826'
2024-12-11T20:12:19,258 DEBUG [RS:0;9fe0640122ec:33749 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-11T20:12:19,259 DEBUG [RS:0;9fe0640122ec:33749 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-11T20:12:19,259 DEBUG [RS:0;9fe0640122ec:33749 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-11T20:12:19,259 INFO  [RS:0;9fe0640122ec:33749 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-11T20:12:19,259 INFO  [RS:0;9fe0640122ec:33749 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-11T20:12:19,363 INFO  [RS:0;9fe0640122ec:33749 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9fe0640122ec%2C33749%2C1733947938826, suffix=, logDir=hdfs://localhost:44595/user/jenkins/test-data/8e0469c0-f728-e9f1-3444-f2c54a669985/WALs/9fe0640122ec,33749,1733947938826, archiveDir=hdfs://localhost:44595/user/jenkins/test-data/8e0469c0-f728-e9f1-3444-f2c54a669985/oldWALs, maxLogs=32
2024-12-11T20:12:19,366 INFO  [RS:0;9fe0640122ec:33749 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9fe0640122ec%2C33749%2C1733947938826.1733947939365
2024-12-11T20:12:19,374 INFO  [RS:0;9fe0640122ec:33749 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/8e0469c0-f728-e9f1-3444-f2c54a669985/WALs/9fe0640122ec,33749,1733947938826/9fe0640122ec%2C33749%2C1733947938826.1733947939365
2024-12-11T20:12:19,381 DEBUG [RS:0;9fe0640122ec:33749 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41319:41319),(127.0.0.1/127.0.0.1:36175:36175)]
2024-12-11T20:12:19,490 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44595/user/jenkins/test-data/8e0469c0-f728-e9f1-3444-f2c54a669985/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-12-11T20:12:19,491 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44595/user/jenkins/test-data/8e0469c0-f728-e9f1-3444-f2c54a669985
2024-12-11T20:12:19,508 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39723 is added to blk_1073741833_1009 (size=32)
2024-12-11T20:12:19,510 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33365 is added to blk_1073741833_1009 (size=32)
2024-12-11T20:12:19,511 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T20:12:19,518 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-11T20:12:19,520 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-11T20:12:19,520 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:12:19,521 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T20:12:19,521 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-11T20:12:19,525 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-11T20:12:19,525 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:12:19,526 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T20:12:19,526 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-11T20:12:19,528 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-11T20:12:19,528 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:12:19,528 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T20:12:19,529 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-11T20:12:19,530 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-11T20:12:19,530 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:12:19,531 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T20:12:19,531 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-11T20:12:19,532 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44595/user/jenkins/test-data/8e0469c0-f728-e9f1-3444-f2c54a669985/data/hbase/meta/1588230740
2024-12-11T20:12:19,532 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44595/user/jenkins/test-data/8e0469c0-f728-e9f1-3444-f2c54a669985/data/hbase/meta/1588230740
2024-12-11T20:12:19,534 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-11T20:12:19,534 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-11T20:12:19,535 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-11T20:12:19,536 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-11T20:12:19,541 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44595/user/jenkins/test-data/8e0469c0-f728-e9f1-3444-f2c54a669985/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-11T20:12:19,541 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=760825, jitterRate=-0.03256222605705261}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-11T20:12:19,542 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1733947939511Initializing all the Stores at 1733947939517 (+6 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733947939518 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733947939518Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733947939518Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733947939518Cleaning up temporary data from old regions at 1733947939534 (+16 ms)Region opened successfully at 1733947939542 (+8 ms)
2024-12-11T20:12:19,543 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-11T20:12:19,543 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-11T20:12:19,543 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-11T20:12:19,543 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-11T20:12:19,543 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-11T20:12:19,544 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-11T20:12:19,544 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733947939543Disabling compacts and flushes for region at 1733947939543Disabling writes for close at 1733947939543Writing region close event to WAL at 1733947939543Closed at 1733947939543
2024-12-11T20:12:19,546 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-11T20:12:19,546 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-12-11T20:12:19,546 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-11T20:12:19,549 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-11T20:12:19,551 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-11T20:12:19,701 DEBUG [9fe0640122ec:45133 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-11T20:12:19,702 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=9fe0640122ec,33749,1733947938826
2024-12-11T20:12:19,704 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9fe0640122ec,33749,1733947938826, state=OPENING
2024-12-11T20:12:19,706 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-11T20:12:19,708 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33749-0x100caf7ea240001, quorum=127.0.0.1:57481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:12:19,708 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45133-0x100caf7ea240000, quorum=127.0.0.1:57481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:12:19,709 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-11T20:12:19,709 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=9fe0640122ec,33749,1733947938826}]
2024-12-11T20:12:19,709 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-11T20:12:19,709 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-11T20:12:19,863 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-11T20:12:19,865 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40241, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-11T20:12:19,875 INFO  [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-12-11T20:12:19,876 INFO  [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-11T20:12:19,879 INFO  [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9fe0640122ec%2C33749%2C1733947938826.meta, suffix=.meta, logDir=hdfs://localhost:44595/user/jenkins/test-data/8e0469c0-f728-e9f1-3444-f2c54a669985/WALs/9fe0640122ec,33749,1733947938826, archiveDir=hdfs://localhost:44595/user/jenkins/test-data/8e0469c0-f728-e9f1-3444-f2c54a669985/oldWALs, maxLogs=32
2024-12-11T20:12:19,881 INFO  [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 9fe0640122ec%2C33749%2C1733947938826.meta.1733947939881.meta
2024-12-11T20:12:19,895 INFO  [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/8e0469c0-f728-e9f1-3444-f2c54a669985/WALs/9fe0640122ec,33749,1733947938826/9fe0640122ec%2C33749%2C1733947938826.meta.1733947939881.meta
2024-12-11T20:12:19,896 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36175:36175),(127.0.0.1/127.0.0.1:41319:41319)]
2024-12-11T20:12:19,897 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-11T20:12:19,897 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-11T20:12:19,897 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-11T20:12:19,897 INFO  [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-11T20:12:19,897 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-11T20:12:19,897 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T20:12:19,898 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-12-11T20:12:19,898 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-12-11T20:12:19,900 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-11T20:12:19,902 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-11T20:12:19,902 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:12:19,903 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T20:12:19,903 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-11T20:12:19,904 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-11T20:12:19,904 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:12:19,905 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T20:12:19,905 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-11T20:12:19,906 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-11T20:12:19,906 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:12:19,907 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T20:12:19,907 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-11T20:12:19,908 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-11T20:12:19,908 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:12:19,908 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T20:12:19,909 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-11T20:12:19,910 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44595/user/jenkins/test-data/8e0469c0-f728-e9f1-3444-f2c54a669985/data/hbase/meta/1588230740
2024-12-11T20:12:19,911 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44595/user/jenkins/test-data/8e0469c0-f728-e9f1-3444-f2c54a669985/data/hbase/meta/1588230740
2024-12-11T20:12:19,913 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-11T20:12:19,913 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-11T20:12:19,914 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-11T20:12:19,917 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-11T20:12:19,918 INFO  [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=739052, jitterRate=-0.060247063636779785}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-11T20:12:19,918 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-12-11T20:12:19,920 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1733947939898Writing region info on filesystem at 1733947939898Initializing all the Stores at 1733947939899 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733947939899Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733947939900 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733947939900Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733947939900Cleaning up temporary data from old regions at 1733947939913 (+13 ms)Running coprocessor post-open hooks at 1733947939918 (+5 ms)Region opened successfully at 1733947939920 (+2 ms)
2024-12-11T20:12:19,921 INFO  [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733947939862
2024-12-11T20:12:19,925 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-11T20:12:19,925 INFO  [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-12-11T20:12:19,927 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=9fe0640122ec,33749,1733947938826
2024-12-11T20:12:19,929 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9fe0640122ec,33749,1733947938826, state=OPEN
2024-12-11T20:12:19,933 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33749-0x100caf7ea240001, quorum=127.0.0.1:57481, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-11T20:12:19,933 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45133-0x100caf7ea240000, quorum=127.0.0.1:57481, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-11T20:12:19,933 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=9fe0640122ec,33749,1733947938826
2024-12-11T20:12:19,933 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-11T20:12:19,933 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-11T20:12:19,938 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-12-11T20:12:19,938 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9fe0640122ec,33749,1733947938826 in 224 msec
2024-12-11T20:12:19,943 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-12-11T20:12:19,943 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 392 msec
2024-12-11T20:12:19,944 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-11T20:12:19,944 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-12-11T20:12:19,946 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-11T20:12:19,946 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9fe0640122ec,33749,1733947938826, seqNum=-1]
2024-12-11T20:12:19,946 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-11T20:12:19,949 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57759, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-11T20:12:19,958 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 902 msec
2024-12-11T20:12:19,958 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733947939958, completionTime=-1
2024-12-11T20:12:19,958 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-11T20:12:19,959 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-12-11T20:12:19,961 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-12-11T20:12:19,961 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733947999961
2024-12-11T20:12:19,961 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733948059961
2024-12-11T20:12:19,961 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec
2024-12-11T20:12:19,961 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9fe0640122ec,45133,1733947938761-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-11T20:12:19,961 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9fe0640122ec,45133,1733947938761-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T20:12:19,962 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9fe0640122ec,45133,1733947938761-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T20:12:19,962 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-9fe0640122ec:45133, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T20:12:19,962 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-11T20:12:19,964 DEBUG [master/9fe0640122ec:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-12-11T20:12:19,969 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-12-11T20:12:19,974 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.099sec
2024-12-11T20:12:19,974 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-11T20:12:19,974 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-11T20:12:19,974 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-11T20:12:19,974 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-11T20:12:19,974 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-11T20:12:19,974 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9fe0640122ec,45133,1733947938761-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-11T20:12:19,974 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9fe0640122ec,45133,1733947938761-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-11T20:12:19,979 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-12-11T20:12:19,979 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-11T20:12:19,979 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9fe0640122ec,45133,1733947938761-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T20:12:20,058 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a89c49a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-11T20:12:20,058 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9fe0640122ec,45133,-1 for getting cluster id
2024-12-11T20:12:20,058 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-12-11T20:12:20,060 DEBUG [HMaster-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c12fb9b2-934e-4e25-b8f8-8881cc86de0c'
2024-12-11T20:12:20,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-12-11T20:12:20,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c12fb9b2-934e-4e25-b8f8-8881cc86de0c"

2024-12-11T20:12:20,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38887dbe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-11T20:12:20,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9fe0640122ec,45133,-1]
2024-12-11T20:12:20,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-12-11T20:12:20,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T20:12:20,064 INFO  [HMaster-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51672, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-12-11T20:12:20,065 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6535c161, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-11T20:12:20,065 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-11T20:12:20,066 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9fe0640122ec,33749,1733947938826, seqNum=-1]
2024-12-11T20:12:20,067 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-11T20:12:20,069 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59416, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-11T20:12:20,071 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=9fe0640122ec,45133,1733947938761
2024-12-11T20:12:20,071 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T20:12:20,075 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-12-11T20:12:20,075 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-12-11T20:12:20,075 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-11T20:12:20,075 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at org.apache.hadoop.hbase.regionserver.wal.TestLogRolling.testLogRollOnDatanodeDeath(TestLogRolling.java:201)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-11T20:12:20,075 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T20:12:20,076 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T20:12:20,076 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-12-11T20:12:20,076 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-11T20:12:20,076 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=216830630, stopped=false
2024-12-11T20:12:20,076 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=9fe0640122ec,45133,1733947938761
2024-12-11T20:12:20,078 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33749-0x100caf7ea240001, quorum=127.0.0.1:57481, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-11T20:12:20,078 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33749-0x100caf7ea240001, quorum=127.0.0.1:57481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:12:20,078 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45133-0x100caf7ea240000, quorum=127.0.0.1:57481, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-11T20:12:20,078 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45133-0x100caf7ea240000, quorum=127.0.0.1:57481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:12:20,078 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-11T20:12:20,078 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-11T20:12:20,079 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at org.apache.hadoop.hbase.regionserver.wal.TestLogRolling.testLogRollOnDatanodeDeath(TestLogRolling.java:201)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-11T20:12:20,079 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33749-0x100caf7ea240001, quorum=127.0.0.1:57481, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-11T20:12:20,079 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T20:12:20,079 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9fe0640122ec,33749,1733947938826' *****
2024-12-11T20:12:20,079 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-12-11T20:12:20,080 INFO  [RS:0;9fe0640122ec:33749 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-11T20:12:20,080 INFO  [RS:0;9fe0640122ec:33749 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-11T20:12:20,080 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-12-11T20:12:20,080 INFO  [RS:0;9fe0640122ec:33749 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-11T20:12:20,080 INFO  [RS:0;9fe0640122ec:33749 {}] regionserver.HRegionServer(959): stopping server 9fe0640122ec,33749,1733947938826
2024-12-11T20:12:20,080 INFO  [RS:0;9fe0640122ec:33749 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-11T20:12:20,080 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45133-0x100caf7ea240000, quorum=127.0.0.1:57481, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-11T20:12:20,080 INFO  [RS:0;9fe0640122ec:33749 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;9fe0640122ec:33749.
2024-12-11T20:12:20,080 DEBUG [RS:0;9fe0640122ec:33749 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-11T20:12:20,080 DEBUG [RS:0;9fe0640122ec:33749 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T20:12:20,080 INFO  [RS:0;9fe0640122ec:33749 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-11T20:12:20,080 INFO  [RS:0;9fe0640122ec:33749 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-11T20:12:20,080 INFO  [RS:0;9fe0640122ec:33749 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-11T20:12:20,081 INFO  [RS:0;9fe0640122ec:33749 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-12-11T20:12:20,081 INFO  [RS:0;9fe0640122ec:33749 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close
2024-12-11T20:12:20,081 DEBUG [RS:0;9fe0640122ec:33749 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740}
2024-12-11T20:12:20,081 DEBUG [RS:0;9fe0640122ec:33749 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-12-11T20:12:20,081 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-11T20:12:20,081 INFO  [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-11T20:12:20,081 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-11T20:12:20,081 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-11T20:12:20,081 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-11T20:12:20,081 INFO  [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=74 B heapSize=1.22 KB
2024-12-11T20:12:20,105 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44595/user/jenkins/test-data/8e0469c0-f728-e9f1-3444-f2c54a669985/data/hbase/meta/1588230740/.tmp/ns/944123dea5614e1597cc3f73dd84a282 is 43, key is default/ns:d/1733947939950/Put/seqid=0
2024-12-11T20:12:20,119 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39723 is added to blk_1073741835_1011 (size=5153)
2024-12-11T20:12:20,120 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33365 is added to blk_1073741835_1011 (size=5153)
2024-12-11T20:12:20,120 INFO  [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:44595/user/jenkins/test-data/8e0469c0-f728-e9f1-3444-f2c54a669985/data/hbase/meta/1588230740/.tmp/ns/944123dea5614e1597cc3f73dd84a282
2024-12-11T20:12:20,138 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44595/user/jenkins/test-data/8e0469c0-f728-e9f1-3444-f2c54a669985/data/hbase/meta/1588230740/.tmp/ns/944123dea5614e1597cc3f73dd84a282 as hdfs://localhost:44595/user/jenkins/test-data/8e0469c0-f728-e9f1-3444-f2c54a669985/data/hbase/meta/1588230740/ns/944123dea5614e1597cc3f73dd84a282
2024-12-11T20:12:20,160 INFO  [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44595/user/jenkins/test-data/8e0469c0-f728-e9f1-3444-f2c54a669985/data/hbase/meta/1588230740/ns/944123dea5614e1597cc3f73dd84a282, entries=2, sequenceid=6, filesize=5.0 K
2024-12-11T20:12:20,161 INFO  [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 80ms, sequenceid=6, compaction requested=false
2024-12-11T20:12:20,161 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-12-11T20:12:20,167 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44595/user/jenkins/test-data/8e0469c0-f728-e9f1-3444-f2c54a669985/data/hbase/meta/1588230740/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1
2024-12-11T20:12:20,168 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-11T20:12:20,168 INFO  [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-11T20:12:20,168 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733947940081Running coprocessor pre-close hooks at 1733947940081Disabling compacts and flushes for region at 1733947940081Disabling writes for close at 1733947940081Obtaining lock to block concurrent updates at 1733947940081Preparing flush snapshotting stores in 1588230740 at 1733947940081Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=74, getHeapSize=1184, getOffHeapSize=0, getCellsCount=2 at 1733947940082 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733947940083 (+1 ms)Flushing 1588230740/ns: creating writer at 1733947940083Flushing 1588230740/ns: appending metadata at 1733947940104 (+21 ms)Flushing 1588230740/ns: closing flushed file at 1733947940104Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6444acb3: reopening flushed file at 1733947940135 (+31 ms)Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 80ms, sequenceid=6, compaction requested=false at 1733947940161 (+26 ms)Writing region close event to WAL at 1733947940163 (+2 ms)Running coprocessor post-close hooks at 1733947940168 (+5 ms)Closed at 1733947940168
2024-12-11T20:12:20,168 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-11T20:12:20,224 INFO  [regionserver/9fe0640122ec:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-12-11T20:12:20,224 INFO  [regionserver/9fe0640122ec:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-12-11T20:12:20,281 INFO  [RS:0;9fe0640122ec:33749 {}] regionserver.HRegionServer(976): stopping server 9fe0640122ec,33749,1733947938826; all regions closed.
2024-12-11T20:12:20,282 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:12:20,282 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:12:20,282 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:12:20,283 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:12:20,283 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:12:20,286 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39723 is added to blk_1073741834_1010 (size=1152)
2024-12-11T20:12:20,287 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33365 is added to blk_1073741834_1010 (size=1152)
2024-12-11T20:12:20,291 DEBUG [RS:0;9fe0640122ec:33749 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/8e0469c0-f728-e9f1-3444-f2c54a669985/oldWALs
2024-12-11T20:12:20,291 INFO  [RS:0;9fe0640122ec:33749 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 9fe0640122ec%2C33749%2C1733947938826.meta:.meta(num 1733947939881)
2024-12-11T20:12:20,291 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:12:20,291 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:12:20,292 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:12:20,292 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:12:20,292 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:12:20,295 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39723 is added to blk_1073741832_1008 (size=93)
2024-12-11T20:12:20,295 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33365 is added to blk_1073741832_1008 (size=93)
2024-12-11T20:12:20,299 DEBUG [RS:0;9fe0640122ec:33749 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/8e0469c0-f728-e9f1-3444-f2c54a669985/oldWALs
2024-12-11T20:12:20,299 INFO  [RS:0;9fe0640122ec:33749 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 9fe0640122ec%2C33749%2C1733947938826:(num 1733947939365)
2024-12-11T20:12:20,299 DEBUG [RS:0;9fe0640122ec:33749 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T20:12:20,299 INFO  [RS:0;9fe0640122ec:33749 {}] regionserver.LeaseManager(133): Closed leases
2024-12-11T20:12:20,299 INFO  [RS:0;9fe0640122ec:33749 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-11T20:12:20,299 INFO  [RS:0;9fe0640122ec:33749 {}] hbase.ChoreService(370): Chore service for: regionserver/9fe0640122ec:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-12-11T20:12:20,300 INFO  [RS:0;9fe0640122ec:33749 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-11T20:12:20,300 INFO  [RS:0;9fe0640122ec:33749 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33749
2024-12-11T20:12:20,300 INFO  [regionserver/9fe0640122ec:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-11T20:12:20,303 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33749-0x100caf7ea240001, quorum=127.0.0.1:57481, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9fe0640122ec,33749,1733947938826
2024-12-11T20:12:20,303 INFO  [RS:0;9fe0640122ec:33749 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-11T20:12:20,303 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45133-0x100caf7ea240000, quorum=127.0.0.1:57481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-11T20:12:20,304 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9fe0640122ec,33749,1733947938826]
2024-12-11T20:12:20,306 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9fe0640122ec,33749,1733947938826 already deleted, retry=false
2024-12-11T20:12:20,306 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 9fe0640122ec,33749,1733947938826 expired; onlineServers=0
2024-12-11T20:12:20,306 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '9fe0640122ec,45133,1733947938761' *****
2024-12-11T20:12:20,306 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-11T20:12:20,306 INFO  [M:0;9fe0640122ec:45133 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-11T20:12:20,306 INFO  [M:0;9fe0640122ec:45133 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-11T20:12:20,307 DEBUG [M:0;9fe0640122ec:45133 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-11T20:12:20,307 DEBUG [M:0;9fe0640122ec:45133 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-11T20:12:20,307 DEBUG [master/9fe0640122ec:0:becomeActiveMaster-HFileCleaner.small.0-1733947939085 {}] cleaner.HFileCleaner(306): Exit Thread[master/9fe0640122ec:0:becomeActiveMaster-HFileCleaner.small.0-1733947939085,5,FailOnTimeoutGroup]
2024-12-11T20:12:20,307 INFO  [M:0;9fe0640122ec:45133 {}] hbase.ChoreService(370): Chore service for: master/9fe0640122ec:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-12-11T20:12:20,307 INFO  [M:0;9fe0640122ec:45133 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-11T20:12:20,307 DEBUG [M:0;9fe0640122ec:45133 {}] master.HMaster(1795): Stopping service threads
2024-12-11T20:12:20,307 INFO  [M:0;9fe0640122ec:45133 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-11T20:12:20,307 INFO  [M:0;9fe0640122ec:45133 {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-11T20:12:20,307 INFO  [M:0;9fe0640122ec:45133 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-11T20:12:20,308 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-11T20:12:20,308 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-11T20:12:20,308 DEBUG [master/9fe0640122ec:0:becomeActiveMaster-HFileCleaner.large.0-1733947939085 {}] cleaner.HFileCleaner(306): Exit Thread[master/9fe0640122ec:0:becomeActiveMaster-HFileCleaner.large.0-1733947939085,5,FailOnTimeoutGroup]
2024-12-11T20:12:20,309 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45133-0x100caf7ea240000, quorum=127.0.0.1:57481, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-11T20:12:20,309 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45133-0x100caf7ea240000, quorum=127.0.0.1:57481, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:12:20,309 DEBUG [M:0;9fe0640122ec:45133 {}] zookeeper.ZKUtil(347): master:45133-0x100caf7ea240000, quorum=127.0.0.1:57481, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-11T20:12:20,310 WARN  [M:0;9fe0640122ec:45133 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-11T20:12:20,310 INFO  [M:0;9fe0640122ec:45133 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44595/user/jenkins/test-data/8e0469c0-f728-e9f1-3444-f2c54a669985/.lastflushedseqids
2024-12-11T20:12:20,326 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33365 is added to blk_1073741836_1012 (size=99)
2024-12-11T20:12:20,327 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39723 is added to blk_1073741836_1012 (size=99)
2024-12-11T20:12:20,328 INFO  [M:0;9fe0640122ec:45133 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-12-11T20:12:20,328 INFO  [M:0;9fe0640122ec:45133 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-11T20:12:20,328 DEBUG [M:0;9fe0640122ec:45133 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-11T20:12:20,328 INFO  [M:0;9fe0640122ec:45133 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T20:12:20,328 DEBUG [M:0;9fe0640122ec:45133 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T20:12:20,329 DEBUG [M:0;9fe0640122ec:45133 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-11T20:12:20,329 DEBUG [M:0;9fe0640122ec:45133 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T20:12:20,329 INFO  [M:0;9fe0640122ec:45133 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.67 KB heapSize=11.34 KB
2024-12-11T20:12:20,348 DEBUG [M:0;9fe0640122ec:45133 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44595/user/jenkins/test-data/8e0469c0-f728-e9f1-3444-f2c54a669985/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b273148d4f584ec4872176a1ade30bf0 is 82, key is hbase:meta,,1/info:regioninfo/1733947939926/Put/seqid=0
2024-12-11T20:12:20,354 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39723 is added to blk_1073741837_1013 (size=5672)
2024-12-11T20:12:20,355 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33365 is added to blk_1073741837_1013 (size=5672)
2024-12-11T20:12:20,355 INFO  [M:0;9fe0640122ec:45133 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:44595/user/jenkins/test-data/8e0469c0-f728-e9f1-3444-f2c54a669985/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b273148d4f584ec4872176a1ade30bf0
2024-12-11T20:12:20,379 DEBUG [M:0;9fe0640122ec:45133 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44595/user/jenkins/test-data/8e0469c0-f728-e9f1-3444-f2c54a669985/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a7771b96c864455089eba40f89226c2c is 240, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1733947939957/Put/seqid=0
2024-12-11T20:12:20,386 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39723 is added to blk_1073741838_1014 (size=5275)
2024-12-11T20:12:20,386 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33365 is added to blk_1073741838_1014 (size=5275)
2024-12-11T20:12:20,389 INFO  [M:0;9fe0640122ec:45133 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:44595/user/jenkins/test-data/8e0469c0-f728-e9f1-3444-f2c54a669985/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a7771b96c864455089eba40f89226c2c
2024-12-11T20:12:20,405 INFO  [RS:0;9fe0640122ec:33749 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-11T20:12:20,405 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33749-0x100caf7ea240001, quorum=127.0.0.1:57481, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-11T20:12:20,405 INFO  [RS:0;9fe0640122ec:33749 {}] regionserver.HRegionServer(1031): Exiting; stopping=9fe0640122ec,33749,1733947938826; zookeeper connection closed.
2024-12-11T20:12:20,405 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33749-0x100caf7ea240001, quorum=127.0.0.1:57481, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-11T20:12:20,405 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3a45321f {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3a45321f
2024-12-11T20:12:20,405 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-12-11T20:12:20,423 DEBUG [M:0;9fe0640122ec:45133 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44595/user/jenkins/test-data/8e0469c0-f728-e9f1-3444-f2c54a669985/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/cd63c1eb76a842f28bdb92eb802ce2ca is 69, key is 9fe0640122ec,33749,1733947938826/rs:state/1733947939181/Put/seqid=0
2024-12-11T20:12:20,430 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39723 is added to blk_1073741839_1015 (size=5156)
2024-12-11T20:12:20,430 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33365 is added to blk_1073741839_1015 (size=5156)
2024-12-11T20:12:20,431 INFO  [M:0;9fe0640122ec:45133 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:44595/user/jenkins/test-data/8e0469c0-f728-e9f1-3444-f2c54a669985/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/cd63c1eb76a842f28bdb92eb802ce2ca
2024-12-11T20:12:20,457 DEBUG [M:0;9fe0640122ec:45133 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44595/user/jenkins/test-data/8e0469c0-f728-e9f1-3444-f2c54a669985/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/16ba4e437e2b4e45a07713e673a1523a is 52, key is load_balancer_on/state:d/1733947940073/Put/seqid=0
2024-12-11T20:12:20,462 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39723 is added to blk_1073741840_1016 (size=5056)
2024-12-11T20:12:20,463 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33365 is added to blk_1073741840_1016 (size=5056)
2024-12-11T20:12:20,463 INFO  [M:0;9fe0640122ec:45133 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:44595/user/jenkins/test-data/8e0469c0-f728-e9f1-3444-f2c54a669985/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/16ba4e437e2b4e45a07713e673a1523a
2024-12-11T20:12:20,471 DEBUG [M:0;9fe0640122ec:45133 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44595/user/jenkins/test-data/8e0469c0-f728-e9f1-3444-f2c54a669985/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b273148d4f584ec4872176a1ade30bf0 as hdfs://localhost:44595/user/jenkins/test-data/8e0469c0-f728-e9f1-3444-f2c54a669985/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b273148d4f584ec4872176a1ade30bf0
2024-12-11T20:12:20,479 INFO  [M:0;9fe0640122ec:45133 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44595/user/jenkins/test-data/8e0469c0-f728-e9f1-3444-f2c54a669985/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b273148d4f584ec4872176a1ade30bf0, entries=8, sequenceid=29, filesize=5.5 K
2024-12-11T20:12:20,480 DEBUG [M:0;9fe0640122ec:45133 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44595/user/jenkins/test-data/8e0469c0-f728-e9f1-3444-f2c54a669985/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a7771b96c864455089eba40f89226c2c as hdfs://localhost:44595/user/jenkins/test-data/8e0469c0-f728-e9f1-3444-f2c54a669985/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a7771b96c864455089eba40f89226c2c
2024-12-11T20:12:20,487 INFO  [M:0;9fe0640122ec:45133 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44595/user/jenkins/test-data/8e0469c0-f728-e9f1-3444-f2c54a669985/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a7771b96c864455089eba40f89226c2c, entries=3, sequenceid=29, filesize=5.2 K
2024-12-11T20:12:20,490 DEBUG [M:0;9fe0640122ec:45133 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44595/user/jenkins/test-data/8e0469c0-f728-e9f1-3444-f2c54a669985/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/cd63c1eb76a842f28bdb92eb802ce2ca as hdfs://localhost:44595/user/jenkins/test-data/8e0469c0-f728-e9f1-3444-f2c54a669985/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/cd63c1eb76a842f28bdb92eb802ce2ca
2024-12-11T20:12:20,498 INFO  [M:0;9fe0640122ec:45133 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44595/user/jenkins/test-data/8e0469c0-f728-e9f1-3444-f2c54a669985/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/cd63c1eb76a842f28bdb92eb802ce2ca, entries=1, sequenceid=29, filesize=5.0 K
2024-12-11T20:12:20,500 DEBUG [M:0;9fe0640122ec:45133 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44595/user/jenkins/test-data/8e0469c0-f728-e9f1-3444-f2c54a669985/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/16ba4e437e2b4e45a07713e673a1523a as hdfs://localhost:44595/user/jenkins/test-data/8e0469c0-f728-e9f1-3444-f2c54a669985/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/16ba4e437e2b4e45a07713e673a1523a
2024-12-11T20:12:20,509 INFO  [M:0;9fe0640122ec:45133 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44595/user/jenkins/test-data/8e0469c0-f728-e9f1-3444-f2c54a669985/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/16ba4e437e2b4e45a07713e673a1523a, entries=1, sequenceid=29, filesize=4.9 K
2024-12-11T20:12:20,511 INFO  [M:0;9fe0640122ec:45133 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 182ms, sequenceid=29, compaction requested=false
2024-12-11T20:12:20,524 INFO  [M:0;9fe0640122ec:45133 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T20:12:20,524 DEBUG [M:0;9fe0640122ec:45133 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733947940328Disabling compacts and flushes for region at 1733947940328Disabling writes for close at 1733947940329 (+1 ms)Obtaining lock to block concurrent updates at 1733947940329Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733947940329Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7850, getHeapSize=11544, getOffHeapSize=0, getCellsCount=36 at 1733947940329Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733947940330 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733947940331 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733947940348 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733947940348Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733947940362 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733947940378 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733947940378Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733947940399 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733947940422 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733947940422Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733947940437 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733947940456 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733947940456Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@516587a1: reopening flushed file at 1733947940470 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3c67d905: reopening flushed file at 1733947940479 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6753ee54: reopening flushed file at 1733947940488 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@665fa134: reopening flushed file at 1733947940498 (+10 ms)Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 182ms, sequenceid=29, compaction requested=false at 1733947940511 (+13 ms)Writing region close event to WAL at 1733947940524 (+13 ms)Closed at 1733947940524
2024-12-11T20:12:20,525 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:12:20,526 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:12:20,526 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:12:20,526 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:12:20,526 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:12:20,530 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33365 is added to blk_1073741830_1006 (size=10311)
2024-12-11T20:12:20,530 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39723 is added to blk_1073741830_1006 (size=10311)
2024-12-11T20:12:20,536 INFO  [M:0;9fe0640122ec:45133 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-12-11T20:12:20,536 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-11T20:12:20,537 INFO  [M:0;9fe0640122ec:45133 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45133
2024-12-11T20:12:20,537 INFO  [M:0;9fe0640122ec:45133 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-11T20:12:20,642 INFO  [M:0;9fe0640122ec:45133 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-11T20:12:20,642 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45133-0x100caf7ea240000, quorum=127.0.0.1:57481, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-11T20:12:20,642 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45133-0x100caf7ea240000, quorum=127.0.0.1:57481, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-11T20:12:20,645 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@156b26b5{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T20:12:20,645 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3dd99d02{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-11T20:12:20,645 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-11T20:12:20,646 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@254e0164{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-11T20:12:20,646 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@30ebe7e3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2a4de8f6-5a88-0e0e-aab1-1c7fba13c249/hadoop.log.dir/,STOPPED}
2024-12-11T20:12:20,647 WARN  [BP-884266491-172.17.0.2-1733947937993 heartbeating to localhost/127.0.0.1:44595 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-11T20:12:20,647 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-11T20:12:20,647 WARN  [BP-884266491-172.17.0.2-1733947937993 heartbeating to localhost/127.0.0.1:44595 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-884266491-172.17.0.2-1733947937993 (Datanode Uuid 37739307-4403-4599-8793-3e4474d318eb) service to localhost/127.0.0.1:44595
2024-12-11T20:12:20,647 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-11T20:12:20,648 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2a4de8f6-5a88-0e0e-aab1-1c7fba13c249/cluster_a95c7252-f29d-d111-d7d7-455ec09db6de/data/data3/current/BP-884266491-172.17.0.2-1733947937993 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T20:12:20,648 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2a4de8f6-5a88-0e0e-aab1-1c7fba13c249/cluster_a95c7252-f29d-d111-d7d7-455ec09db6de/data/data4/current/BP-884266491-172.17.0.2-1733947937993 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T20:12:20,648 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-11T20:12:20,651 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7bd218c7{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T20:12:20,652 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5a9b9566{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-11T20:12:20,652 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-11T20:12:20,652 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1d281952{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-11T20:12:20,652 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@b5aedfa{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2a4de8f6-5a88-0e0e-aab1-1c7fba13c249/hadoop.log.dir/,STOPPED}
2024-12-11T20:12:20,654 WARN  [BP-884266491-172.17.0.2-1733947937993 heartbeating to localhost/127.0.0.1:44595 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-11T20:12:20,654 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-11T20:12:20,654 WARN  [BP-884266491-172.17.0.2-1733947937993 heartbeating to localhost/127.0.0.1:44595 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-884266491-172.17.0.2-1733947937993 (Datanode Uuid e2a77dfe-2623-4901-975a-c3c2425f2e9e) service to localhost/127.0.0.1:44595
2024-12-11T20:12:20,654 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-11T20:12:20,655 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2a4de8f6-5a88-0e0e-aab1-1c7fba13c249/cluster_a95c7252-f29d-d111-d7d7-455ec09db6de/data/data1/current/BP-884266491-172.17.0.2-1733947937993 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T20:12:20,655 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2a4de8f6-5a88-0e0e-aab1-1c7fba13c249/cluster_a95c7252-f29d-d111-d7d7-455ec09db6de/data/data2/current/BP-884266491-172.17.0.2-1733947937993 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T20:12:20,655 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-11T20:12:20,662 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@245f97f3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-11T20:12:20,662 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@219d080{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-11T20:12:20,662 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-11T20:12:20,662 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@10885b70{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-11T20:12:20,663 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6c443180{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2a4de8f6-5a88-0e0e-aab1-1c7fba13c249/hadoop.log.dir/,STOPPED}
2024-12-11T20:12:20,669 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-12-11T20:12:20,685 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-12-11T20:12:20,685 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-11T20:12:20,686 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2a4de8f6-5a88-0e0e-aab1-1c7fba13c249/hadoop.log.dir so I do NOT create it in target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad
2024-12-11T20:12:20,686 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2a4de8f6-5a88-0e0e-aab1-1c7fba13c249/hadoop.tmp.dir so I do NOT create it in target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad
2024-12-11T20:12:20,686 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/cluster_f21598ea-49df-c0f8-b759-6a4f22597d28, deleteOnExit=true
2024-12-11T20:12:20,686 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-12-11T20:12:20,686 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/test.cache.data in system properties and HBase conf
2024-12-11T20:12:20,686 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/hadoop.tmp.dir in system properties and HBase conf
2024-12-11T20:12:20,686 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/hadoop.log.dir in system properties and HBase conf
2024-12-11T20:12:20,686 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-11T20:12:20,686 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-11T20:12:20,686 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-12-11T20:12:20,687 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-11T20:12:20,687 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-11T20:12:20,687 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-11T20:12:20,687 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-11T20:12:20,687 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-11T20:12:20,687 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-11T20:12:20,687 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-11T20:12:20,687 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-11T20:12:20,687 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-11T20:12:20,688 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-11T20:12:20,688 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/nfs.dump.dir in system properties and HBase conf
2024-12-11T20:12:20,688 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/java.io.tmpdir in system properties and HBase conf
2024-12-11T20:12:20,688 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-11T20:12:20,688 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-11T20:12:20,688 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-11T20:12:20,711 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-11T20:12:20,811 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T20:12:20,819 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-11T20:12:20,822 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-11T20:12:20,822 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-11T20:12:20,822 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-11T20:12:20,823 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T20:12:20,823 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@73b9709e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/hadoop.log.dir/,AVAILABLE}
2024-12-11T20:12:20,824 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2fd186ec{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-11T20:12:20,942 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@d607c51{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/java.io.tmpdir/jetty-localhost-44323-hadoop-hdfs-3_4_1-tests_jar-_-any-10026413276517063570/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-11T20:12:20,943 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7ae0a20f{HTTP/1.1, (http/1.1)}{localhost:44323}
2024-12-11T20:12:20,943 INFO  [Time-limited test {}] server.Server(415): Started @104539ms
2024-12-11T20:12:20,958 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-11T20:12:21,043 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T20:12:21,047 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-11T20:12:21,047 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-11T20:12:21,047 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-11T20:12:21,047 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-11T20:12:21,048 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6787773a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/hadoop.log.dir/,AVAILABLE}
2024-12-11T20:12:21,048 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@ee6b493{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-11T20:12:21,195 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3e26ba04{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/java.io.tmpdir/jetty-localhost-35101-hadoop-hdfs-3_4_1-tests_jar-_-any-5723649984301803637/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T20:12:21,196 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7f515a91{HTTP/1.1, (http/1.1)}{localhost:35101}
2024-12-11T20:12:21,196 INFO  [Time-limited test {}] server.Server(415): Started @104792ms
2024-12-11T20:12:21,198 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-11T20:12:21,222 INFO  [regionserver/9fe0640122ec:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-11T20:12:21,251 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T20:12:21,256 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-11T20:12:21,261 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-11T20:12:21,261 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-11T20:12:21,261 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-11T20:12:21,261 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3b918d2a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/hadoop.log.dir/,AVAILABLE}
2024-12-11T20:12:21,262 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@20aa2ea7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-11T20:12:21,310 WARN  [Thread-659 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/cluster_f21598ea-49df-c0f8-b759-6a4f22597d28/data/data1/current/BP-773969328-172.17.0.2-1733947940734/current, will proceed with Du for space computation calculation, 
2024-12-11T20:12:21,310 WARN  [Thread-660 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/cluster_f21598ea-49df-c0f8-b759-6a4f22597d28/data/data2/current/BP-773969328-172.17.0.2-1733947940734/current, will proceed with Du for space computation calculation, 
2024-12-11T20:12:21,329 WARN  [Thread-638 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-11T20:12:21,333 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xdbe61f0adbc4895b with lease ID 0x66af004492a15cc9: Processing first storage report for DS-c09ca836-bfa5-489f-bed7-23a82edbbc28 from datanode DatanodeRegistration(127.0.0.1:34893, datanodeUuid=ab505b67-ff6c-4d4a-96e5-79bfe9f4b042, infoPort=45941, infoSecurePort=0, ipcPort=41499, storageInfo=lv=-57;cid=testClusterID;nsid=1349632812;c=1733947940734)
2024-12-11T20:12:21,333 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xdbe61f0adbc4895b with lease ID 0x66af004492a15cc9: from storage DS-c09ca836-bfa5-489f-bed7-23a82edbbc28 node DatanodeRegistration(127.0.0.1:34893, datanodeUuid=ab505b67-ff6c-4d4a-96e5-79bfe9f4b042, infoPort=45941, infoSecurePort=0, ipcPort=41499, storageInfo=lv=-57;cid=testClusterID;nsid=1349632812;c=1733947940734), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-11T20:12:21,333 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xdbe61f0adbc4895b with lease ID 0x66af004492a15cc9: Processing first storage report for DS-3d2eb58b-f8c1-41c6-ba40-f862e2b958ad from datanode DatanodeRegistration(127.0.0.1:34893, datanodeUuid=ab505b67-ff6c-4d4a-96e5-79bfe9f4b042, infoPort=45941, infoSecurePort=0, ipcPort=41499, storageInfo=lv=-57;cid=testClusterID;nsid=1349632812;c=1733947940734)
2024-12-11T20:12:21,333 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xdbe61f0adbc4895b with lease ID 0x66af004492a15cc9: from storage DS-3d2eb58b-f8c1-41c6-ba40-f862e2b958ad node DatanodeRegistration(127.0.0.1:34893, datanodeUuid=ab505b67-ff6c-4d4a-96e5-79bfe9f4b042, infoPort=45941, infoSecurePort=0, ipcPort=41499, storageInfo=lv=-57;cid=testClusterID;nsid=1349632812;c=1733947940734), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T20:12:21,384 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6222f16{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/java.io.tmpdir/jetty-localhost-43811-hadoop-hdfs-3_4_1-tests_jar-_-any-2063826524840620877/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T20:12:21,384 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5d9b1613{HTTP/1.1, (http/1.1)}{localhost:43811}
2024-12-11T20:12:21,384 INFO  [Time-limited test {}] server.Server(415): Started @104980ms
2024-12-11T20:12:21,387 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-11T20:12:21,492 WARN  [Thread-686 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/cluster_f21598ea-49df-c0f8-b759-6a4f22597d28/data/data4/current/BP-773969328-172.17.0.2-1733947940734/current, will proceed with Du for space computation calculation, 
2024-12-11T20:12:21,492 WARN  [Thread-685 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/cluster_f21598ea-49df-c0f8-b759-6a4f22597d28/data/data3/current/BP-773969328-172.17.0.2-1733947940734/current, will proceed with Du for space computation calculation, 
2024-12-11T20:12:21,510 WARN  [Thread-674 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-11T20:12:21,513 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8c34d7d3fb9f31a9 with lease ID 0x66af004492a15cca: Processing first storage report for DS-a9e47326-b9f8-4a53-b473-bfa84b160cb2 from datanode DatanodeRegistration(127.0.0.1:36207, datanodeUuid=669f8eed-2f96-4854-8c9a-69c5c7a3d0c2, infoPort=35449, infoSecurePort=0, ipcPort=37621, storageInfo=lv=-57;cid=testClusterID;nsid=1349632812;c=1733947940734)
2024-12-11T20:12:21,513 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8c34d7d3fb9f31a9 with lease ID 0x66af004492a15cca: from storage DS-a9e47326-b9f8-4a53-b473-bfa84b160cb2 node DatanodeRegistration(127.0.0.1:36207, datanodeUuid=669f8eed-2f96-4854-8c9a-69c5c7a3d0c2, infoPort=35449, infoSecurePort=0, ipcPort=37621, storageInfo=lv=-57;cid=testClusterID;nsid=1349632812;c=1733947940734), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-11T20:12:21,513 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8c34d7d3fb9f31a9 with lease ID 0x66af004492a15cca: Processing first storage report for DS-4f4193c3-6231-4e76-9dbd-6338e3774ec4 from datanode DatanodeRegistration(127.0.0.1:36207, datanodeUuid=669f8eed-2f96-4854-8c9a-69c5c7a3d0c2, infoPort=35449, infoSecurePort=0, ipcPort=37621, storageInfo=lv=-57;cid=testClusterID;nsid=1349632812;c=1733947940734)
2024-12-11T20:12:21,514 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8c34d7d3fb9f31a9 with lease ID 0x66af004492a15cca: from storage DS-4f4193c3-6231-4e76-9dbd-6338e3774ec4 node DatanodeRegistration(127.0.0.1:36207, datanodeUuid=669f8eed-2f96-4854-8c9a-69c5c7a3d0c2, infoPort=35449, infoSecurePort=0, ipcPort=37621, storageInfo=lv=-57;cid=testClusterID;nsid=1349632812;c=1733947940734), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T20:12:21,527 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad
2024-12-11T20:12:21,532 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/cluster_f21598ea-49df-c0f8-b759-6a4f22597d28/zookeeper_0, clientPort=56424, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/cluster_f21598ea-49df-c0f8-b759-6a4f22597d28/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/cluster_f21598ea-49df-c0f8-b759-6a4f22597d28/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-11T20:12:21,532 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=56424
2024-12-11T20:12:21,533 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T20:12:21,535 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T20:12:21,551 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36207 is added to blk_1073741825_1001 (size=7)
2024-12-11T20:12:21,551 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34893 is added to blk_1073741825_1001 (size=7)
2024-12-11T20:12:21,953 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f with version=8
2024-12-11T20:12:21,953 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/hbase-staging
2024-12-11T20:12:21,957 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/9fe0640122ec:0 server-side Connection retries=45
2024-12-11T20:12:21,957 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-11T20:12:21,957 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-11T20:12:21,957 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-11T20:12:21,957 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-11T20:12:21,957 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-11T20:12:21,957 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-12-11T20:12:21,957 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-11T20:12:21,958 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38011
2024-12-11T20:12:21,960 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38011 connecting to ZooKeeper ensemble=127.0.0.1:56424
2024-12-11T20:12:21,966 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:380110x0, quorum=127.0.0.1:56424, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-11T20:12:21,967 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38011-0x100caf7f5180000 connected
2024-12-11T20:12:21,992 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T20:12:21,995 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T20:12:21,997 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38011-0x100caf7f5180000, quorum=127.0.0.1:56424, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-11T20:12:21,997 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f, hbase.cluster.distributed=false
2024-12-11T20:12:21,999 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38011-0x100caf7f5180000, quorum=127.0.0.1:56424, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-11T20:12:22,003 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38011
2024-12-11T20:12:22,004 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38011
2024-12-11T20:12:22,005 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38011
2024-12-11T20:12:22,005 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38011
2024-12-11T20:12:22,008 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38011
2024-12-11T20:12:22,024 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/9fe0640122ec:0 server-side Connection retries=45
2024-12-11T20:12:22,024 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-11T20:12:22,025 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-11T20:12:22,025 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-11T20:12:22,025 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-11T20:12:22,025 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-11T20:12:22,025 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-11T20:12:22,025 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-11T20:12:22,026 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42593
2024-12-11T20:12:22,027 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42593 connecting to ZooKeeper ensemble=127.0.0.1:56424
2024-12-11T20:12:22,028 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T20:12:22,030 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T20:12:22,035 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:425930x0, quorum=127.0.0.1:56424, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-11T20:12:22,035 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:425930x0, quorum=127.0.0.1:56424, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-11T20:12:22,035 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42593-0x100caf7f5180001 connected
2024-12-11T20:12:22,036 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-11T20:12:22,036 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-11T20:12:22,037 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42593-0x100caf7f5180001, quorum=127.0.0.1:56424, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-11T20:12:22,038 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42593-0x100caf7f5180001, quorum=127.0.0.1:56424, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-11T20:12:22,039 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42593
2024-12-11T20:12:22,040 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42593
2024-12-11T20:12:22,040 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42593
2024-12-11T20:12:22,041 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42593
2024-12-11T20:12:22,041 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42593
2024-12-11T20:12:22,054 DEBUG [M:0;9fe0640122ec:38011 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;9fe0640122ec:38011
2024-12-11T20:12:22,055 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/9fe0640122ec,38011,1733947941956
2024-12-11T20:12:22,057 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42593-0x100caf7f5180001, quorum=127.0.0.1:56424, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-11T20:12:22,057 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38011-0x100caf7f5180000, quorum=127.0.0.1:56424, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-11T20:12:22,058 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38011-0x100caf7f5180000, quorum=127.0.0.1:56424, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/9fe0640122ec,38011,1733947941956
2024-12-11T20:12:22,059 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38011-0x100caf7f5180000, quorum=127.0.0.1:56424, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:12:22,059 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42593-0x100caf7f5180001, quorum=127.0.0.1:56424, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-11T20:12:22,060 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42593-0x100caf7f5180001, quorum=127.0.0.1:56424, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:12:22,060 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38011-0x100caf7f5180000, quorum=127.0.0.1:56424, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-11T20:12:22,061 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/9fe0640122ec,38011,1733947941956 from backup master directory
2024-12-11T20:12:22,063 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42593-0x100caf7f5180001, quorum=127.0.0.1:56424, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-11T20:12:22,063 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38011-0x100caf7f5180000, quorum=127.0.0.1:56424, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/9fe0640122ec,38011,1733947941956
2024-12-11T20:12:22,063 WARN  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-11T20:12:22,063 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=9fe0640122ec,38011,1733947941956
2024-12-11T20:12:22,063 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38011-0x100caf7f5180000, quorum=127.0.0.1:56424, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-11T20:12:22,068 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/hbase.id] with ID: c629bce1-e155-4122-ab22-c357595bb826
2024-12-11T20:12:22,069 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/.tmp/hbase.id
2024-12-11T20:12:22,078 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34893 is added to blk_1073741826_1002 (size=42)
2024-12-11T20:12:22,079 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36207 is added to blk_1073741826_1002 (size=42)
2024-12-11T20:12:22,080 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/.tmp/hbase.id]:[hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/hbase.id]
2024-12-11T20:12:22,096 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T20:12:22,096 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-12-11T20:12:22,098 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms.
2024-12-11T20:12:22,101 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42593-0x100caf7f5180001, quorum=127.0.0.1:56424, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:12:22,101 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38011-0x100caf7f5180000, quorum=127.0.0.1:56424, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:12:22,113 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36207 is added to blk_1073741827_1003 (size=196)
2024-12-11T20:12:22,113 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34893 is added to blk_1073741827_1003 (size=196)
2024-12-11T20:12:22,115 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-11T20:12:22,116 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-11T20:12:22,116 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-11T20:12:22,128 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34893 is added to blk_1073741828_1004 (size=1189)
2024-12-11T20:12:22,130 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36207 is added to blk_1073741828_1004 (size=1189)
2024-12-11T20:12:22,130 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/MasterData/data/master/store
2024-12-11T20:12:22,146 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36207 is added to blk_1073741829_1005 (size=34)
2024-12-11T20:12:22,148 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34893 is added to blk_1073741829_1005 (size=34)
2024-12-11T20:12:22,148 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T20:12:22,149 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-11T20:12:22,149 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T20:12:22,149 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T20:12:22,149 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-11T20:12:22,149 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T20:12:22,149 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T20:12:22,149 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733947942148Disabling compacts and flushes for region at 1733947942149 (+1 ms)Disabling writes for close at 1733947942149Writing region close event to WAL at 1733947942149Closed at 1733947942149
2024-12-11T20:12:22,150 WARN  [master/9fe0640122ec:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/MasterData/data/master/store/.initializing
2024-12-11T20:12:22,150 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/MasterData/WALs/9fe0640122ec,38011,1733947941956
2024-12-11T20:12:22,153 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9fe0640122ec%2C38011%2C1733947941956, suffix=, logDir=hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/MasterData/WALs/9fe0640122ec,38011,1733947941956, archiveDir=hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/MasterData/oldWALs, maxLogs=10
2024-12-11T20:12:22,154 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9fe0640122ec%2C38011%2C1733947941956.1733947942154
2024-12-11T20:12:22,156 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-11T20:12:22,156 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-12-11T20:12:22,157 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-12-11T20:12:22,158 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling
2024-12-11T20:12:22,170 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/MasterData/WALs/9fe0640122ec,38011,1733947941956/9fe0640122ec%2C38011%2C1733947941956.1733947942154
2024-12-11T20:12:22,179 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35449:35449),(127.0.0.1/127.0.0.1:45941:45941)]
2024-12-11T20:12:22,180 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-11T20:12:22,180 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T20:12:22,180 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T20:12:22,180 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T20:12:22,185 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-11T20:12:22,187 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-11T20:12:22,187 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:12:22,188 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T20:12:22,188 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-11T20:12:22,190 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-11T20:12:22,190 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:12:22,190 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-11T20:12:22,191 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-11T20:12:22,192 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-11T20:12:22,192 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:12:22,193 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-11T20:12:22,193 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-11T20:12:22,195 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-11T20:12:22,195 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:12:22,195 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-11T20:12:22,196 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T20:12:22,197 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-11T20:12:22,197 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-11T20:12:22,199 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T20:12:22,199 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T20:12:22,200 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-11T20:12:22,202 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T20:12:22,205 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-11T20:12:22,206 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=735837, jitterRate=-0.06433533132076263}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-11T20:12:22,207 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1733947942181Initializing all the Stores at 1733947942182 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733947942182Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733947942184 (+2 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733947942184Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733947942184Cleaning up temporary data from old regions at 1733947942199 (+15 ms)Region opened successfully at 1733947942207 (+8 ms)
2024-12-11T20:12:22,208 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-11T20:12:22,213 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a9f9568, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9fe0640122ec/172.17.0.2:0
2024-12-11T20:12:22,214 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-12-11T20:12:22,214 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-11T20:12:22,214 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-11T20:12:22,215 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-11T20:12:22,215 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-12-11T20:12:22,216 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-12-11T20:12:22,216 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-11T20:12:22,222 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-11T20:12:22,223 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38011-0x100caf7f5180000, quorum=127.0.0.1:56424, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-11T20:12:22,225 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-12-11T20:12:22,225 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-11T20:12:22,226 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38011-0x100caf7f5180000, quorum=127.0.0.1:56424, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-11T20:12:22,227 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-12-11T20:12:22,228 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-11T20:12:22,229 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38011-0x100caf7f5180000, quorum=127.0.0.1:56424, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-11T20:12:22,230 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-12-11T20:12:22,232 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38011-0x100caf7f5180000, quorum=127.0.0.1:56424, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-11T20:12:22,233 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-12-11T20:12:22,236 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38011-0x100caf7f5180000, quorum=127.0.0.1:56424, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-11T20:12:22,237 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-11T20:12:22,239 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38011-0x100caf7f5180000, quorum=127.0.0.1:56424, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-11T20:12:22,239 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38011-0x100caf7f5180000, quorum=127.0.0.1:56424, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:12:22,239 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42593-0x100caf7f5180001, quorum=127.0.0.1:56424, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-11T20:12:22,239 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42593-0x100caf7f5180001, quorum=127.0.0.1:56424, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:12:22,240 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=9fe0640122ec,38011,1733947941956, sessionid=0x100caf7f5180000, setting cluster-up flag (Was=false)
2024-12-11T20:12:22,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38011-0x100caf7f5180000, quorum=127.0.0.1:56424, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:12:22,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42593-0x100caf7f5180001, quorum=127.0.0.1:56424, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:12:22,248 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-11T20:12:22,249 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9fe0640122ec,38011,1733947941956
2024-12-11T20:12:22,253 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38011-0x100caf7f5180000, quorum=127.0.0.1:56424, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:12:22,253 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42593-0x100caf7f5180001, quorum=127.0.0.1:56424, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:12:22,259 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-11T20:12:22,261 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9fe0640122ec,38011,1733947941956
2024-12-11T20:12:22,265 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-12-11T20:12:22,269 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-12-11T20:12:22,270 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-12-11T20:12:22,270 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-11T20:12:22,270 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 9fe0640122ec,38011,1733947941956
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-11T20:12:22,272 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/9fe0640122ec:0, corePoolSize=5, maxPoolSize=5
2024-12-11T20:12:22,273 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/9fe0640122ec:0, corePoolSize=5, maxPoolSize=5
2024-12-11T20:12:22,273 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/9fe0640122ec:0, corePoolSize=5, maxPoolSize=5
2024-12-11T20:12:22,273 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/9fe0640122ec:0, corePoolSize=5, maxPoolSize=5
2024-12-11T20:12:22,273 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/9fe0640122ec:0, corePoolSize=10, maxPoolSize=10
2024-12-11T20:12:22,273 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:12:22,273 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/9fe0640122ec:0, corePoolSize=2, maxPoolSize=2
2024-12-11T20:12:22,273 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:12:22,274 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733947972274
2024-12-11T20:12:22,274 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-11T20:12:22,274 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-11T20:12:22,274 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-11T20:12:22,274 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-11T20:12:22,275 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-11T20:12:22,275 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-11T20:12:22,275 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-11T20:12:22,275 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-11T20:12:22,276 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-11T20:12:22,276 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-11T20:12:22,276 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-11T20:12:22,276 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-12-11T20:12:22,276 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-11T20:12:22,276 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-11T20:12:22,277 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/9fe0640122ec:0:becomeActiveMaster-HFileCleaner.large.0-1733947942277,5,FailOnTimeoutGroup]
2024-12-11T20:12:22,277 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/9fe0640122ec:0:becomeActiveMaster-HFileCleaner.small.0-1733947942277,5,FailOnTimeoutGroup]
2024-12-11T20:12:22,277 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-11T20:12:22,277 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-11T20:12:22,278 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-11T20:12:22,278 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:12:22,278 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-11T20:12:22,278 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-11T20:12:22,307 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34893 is added to blk_1073741831_1007 (size=1321)
2024-12-11T20:12:22,308 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36207 is added to blk_1073741831_1007 (size=1321)
2024-12-11T20:12:22,309 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-12-11T20:12:22,310 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f
2024-12-11T20:12:22,313 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:12:22,319 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:12:22,327 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36207 is added to blk_1073741832_1008 (size=32)
2024-12-11T20:12:22,327 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34893 is added to blk_1073741832_1008 (size=32)
2024-12-11T20:12:22,328 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T20:12:22,331 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-11T20:12:22,333 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-11T20:12:22,333 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:12:22,334 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T20:12:22,334 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-11T20:12:22,336 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-11T20:12:22,336 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:12:22,336 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T20:12:22,337 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-11T20:12:22,338 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-11T20:12:22,338 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:12:22,339 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T20:12:22,339 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-11T20:12:22,341 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-11T20:12:22,341 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:12:22,341 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T20:12:22,341 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-11T20:12:22,342 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/hbase/meta/1588230740
2024-12-11T20:12:22,343 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/hbase/meta/1588230740
2024-12-11T20:12:22,343 INFO  [RS:0;9fe0640122ec:42593 {}] regionserver.HRegionServer(746): ClusterId : c629bce1-e155-4122-ab22-c357595bb826
2024-12-11T20:12:22,343 DEBUG [RS:0;9fe0640122ec:42593 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-11T20:12:22,345 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-11T20:12:22,345 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-11T20:12:22,346 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-11T20:12:22,346 DEBUG [RS:0;9fe0640122ec:42593 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-11T20:12:22,346 DEBUG [RS:0;9fe0640122ec:42593 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-11T20:12:22,347 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-11T20:12:22,349 DEBUG [RS:0;9fe0640122ec:42593 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-11T20:12:22,350 DEBUG [RS:0;9fe0640122ec:42593 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@397196d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9fe0640122ec/172.17.0.2:0
2024-12-11T20:12:22,350 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-11T20:12:22,350 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=787677, jitterRate=0.0015833228826522827}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-11T20:12:22,352 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1733947942328Initializing all the Stores at 1733947942330 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733947942330Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733947942331 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733947942331Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733947942331Cleaning up temporary data from old regions at 1733947942345 (+14 ms)Region opened successfully at 1733947942352 (+7 ms)
2024-12-11T20:12:22,352 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-11T20:12:22,352 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-11T20:12:22,352 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-11T20:12:22,352 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-11T20:12:22,352 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-11T20:12:22,357 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-11T20:12:22,357 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733947942352Disabling compacts and flushes for region at 1733947942352Disabling writes for close at 1733947942352Writing region close event to WAL at 1733947942357 (+5 ms)Closed at 1733947942357
2024-12-11T20:12:22,359 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-11T20:12:22,359 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-12-11T20:12:22,359 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-11T20:12:22,361 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-11T20:12:22,362 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-11T20:12:22,366 DEBUG [RS:0;9fe0640122ec:42593 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;9fe0640122ec:42593
2024-12-11T20:12:22,366 INFO  [RS:0;9fe0640122ec:42593 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-11T20:12:22,366 INFO  [RS:0;9fe0640122ec:42593 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-11T20:12:22,366 DEBUG [RS:0;9fe0640122ec:42593 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-11T20:12:22,367 INFO  [RS:0;9fe0640122ec:42593 {}] regionserver.HRegionServer(2659): reportForDuty to master=9fe0640122ec,38011,1733947941956 with port=42593, startcode=1733947942024
2024-12-11T20:12:22,367 DEBUG [RS:0;9fe0640122ec:42593 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-11T20:12:22,369 INFO  [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46235, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService
2024-12-11T20:12:22,370 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38011 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9fe0640122ec,42593,1733947942024
2024-12-11T20:12:22,370 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38011 {}] master.ServerManager(517): Registering regionserver=9fe0640122ec,42593,1733947942024
2024-12-11T20:12:22,372 DEBUG [RS:0;9fe0640122ec:42593 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f
2024-12-11T20:12:22,372 DEBUG [RS:0;9fe0640122ec:42593 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40619
2024-12-11T20:12:22,372 DEBUG [RS:0;9fe0640122ec:42593 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-11T20:12:22,374 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38011-0x100caf7f5180000, quorum=127.0.0.1:56424, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-11T20:12:22,374 DEBUG [RS:0;9fe0640122ec:42593 {}] zookeeper.ZKUtil(111): regionserver:42593-0x100caf7f5180001, quorum=127.0.0.1:56424, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9fe0640122ec,42593,1733947942024
2024-12-11T20:12:22,374 WARN  [RS:0;9fe0640122ec:42593 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-11T20:12:22,374 INFO  [RS:0;9fe0640122ec:42593 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-11T20:12:22,375 DEBUG [RS:0;9fe0640122ec:42593 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024
2024-12-11T20:12:22,375 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9fe0640122ec,42593,1733947942024]
2024-12-11T20:12:22,379 INFO  [RS:0;9fe0640122ec:42593 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-11T20:12:22,383 INFO  [RS:0;9fe0640122ec:42593 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-11T20:12:22,383 INFO  [RS:0;9fe0640122ec:42593 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-11T20:12:22,383 INFO  [RS:0;9fe0640122ec:42593 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-11T20:12:22,384 INFO  [RS:0;9fe0640122ec:42593 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-11T20:12:22,386 INFO  [RS:0;9fe0640122ec:42593 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-11T20:12:22,386 INFO  [RS:0;9fe0640122ec:42593 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-11T20:12:22,386 DEBUG [RS:0;9fe0640122ec:42593 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:12:22,386 DEBUG [RS:0;9fe0640122ec:42593 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:12:22,386 DEBUG [RS:0;9fe0640122ec:42593 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:12:22,386 DEBUG [RS:0;9fe0640122ec:42593 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:12:22,386 DEBUG [RS:0;9fe0640122ec:42593 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:12:22,387 DEBUG [RS:0;9fe0640122ec:42593 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9fe0640122ec:0, corePoolSize=2, maxPoolSize=2
2024-12-11T20:12:22,387 DEBUG [RS:0;9fe0640122ec:42593 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:12:22,387 DEBUG [RS:0;9fe0640122ec:42593 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:12:22,387 DEBUG [RS:0;9fe0640122ec:42593 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:12:22,387 DEBUG [RS:0;9fe0640122ec:42593 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:12:22,387 DEBUG [RS:0;9fe0640122ec:42593 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:12:22,387 DEBUG [RS:0;9fe0640122ec:42593 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:12:22,387 DEBUG [RS:0;9fe0640122ec:42593 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9fe0640122ec:0, corePoolSize=3, maxPoolSize=3
2024-12-11T20:12:22,387 DEBUG [RS:0;9fe0640122ec:42593 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9fe0640122ec:0, corePoolSize=3, maxPoolSize=3
2024-12-11T20:12:22,396 INFO  [RS:0;9fe0640122ec:42593 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-11T20:12:22,397 INFO  [RS:0;9fe0640122ec:42593 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-11T20:12:22,397 INFO  [RS:0;9fe0640122ec:42593 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-11T20:12:22,397 INFO  [RS:0;9fe0640122ec:42593 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-11T20:12:22,397 INFO  [RS:0;9fe0640122ec:42593 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-11T20:12:22,397 INFO  [RS:0;9fe0640122ec:42593 {}] hbase.ChoreService(168): Chore ScheduledChore name=9fe0640122ec,42593,1733947942024-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-11T20:12:22,414 INFO  [RS:0;9fe0640122ec:42593 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-11T20:12:22,414 INFO  [RS:0;9fe0640122ec:42593 {}] hbase.ChoreService(168): Chore ScheduledChore name=9fe0640122ec,42593,1733947942024-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-11T20:12:22,414 INFO  [RS:0;9fe0640122ec:42593 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T20:12:22,414 INFO  [RS:0;9fe0640122ec:42593 {}] regionserver.Replication(171): 9fe0640122ec,42593,1733947942024 started
2024-12-11T20:12:22,431 INFO  [RS:0;9fe0640122ec:42593 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T20:12:22,431 INFO  [RS:0;9fe0640122ec:42593 {}] regionserver.HRegionServer(1482): Serving as 9fe0640122ec,42593,1733947942024, RpcServer on 9fe0640122ec/172.17.0.2:42593, sessionid=0x100caf7f5180001
2024-12-11T20:12:22,431 DEBUG [RS:0;9fe0640122ec:42593 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-11T20:12:22,431 DEBUG [RS:0;9fe0640122ec:42593 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9fe0640122ec,42593,1733947942024
2024-12-11T20:12:22,431 DEBUG [RS:0;9fe0640122ec:42593 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9fe0640122ec,42593,1733947942024'
2024-12-11T20:12:22,431 DEBUG [RS:0;9fe0640122ec:42593 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-11T20:12:22,432 DEBUG [RS:0;9fe0640122ec:42593 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-11T20:12:22,433 DEBUG [RS:0;9fe0640122ec:42593 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-11T20:12:22,433 DEBUG [RS:0;9fe0640122ec:42593 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-11T20:12:22,433 DEBUG [RS:0;9fe0640122ec:42593 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9fe0640122ec,42593,1733947942024
2024-12-11T20:12:22,433 DEBUG [RS:0;9fe0640122ec:42593 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9fe0640122ec,42593,1733947942024'
2024-12-11T20:12:22,433 DEBUG [RS:0;9fe0640122ec:42593 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-11T20:12:22,434 DEBUG [RS:0;9fe0640122ec:42593 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-11T20:12:22,437 DEBUG [RS:0;9fe0640122ec:42593 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-11T20:12:22,437 INFO  [RS:0;9fe0640122ec:42593 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-11T20:12:22,437 INFO  [RS:0;9fe0640122ec:42593 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-11T20:12:22,513 WARN  [9fe0640122ec:38011 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-12-11T20:12:22,540 INFO  [RS:0;9fe0640122ec:42593 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9fe0640122ec%2C42593%2C1733947942024, suffix=, logDir=hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024, archiveDir=hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/oldWALs, maxLogs=32
2024-12-11T20:12:22,542 INFO  [RS:0;9fe0640122ec:42593 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9fe0640122ec%2C42593%2C1733947942024.1733947942541
2024-12-11T20:12:22,554 INFO  [RS:0;9fe0640122ec:42593 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.1733947942541
2024-12-11T20:12:22,556 DEBUG [RS:0;9fe0640122ec:42593 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35449:35449),(127.0.0.1/127.0.0.1:45941:45941)]
2024-12-11T20:12:22,763 DEBUG [9fe0640122ec:38011 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-11T20:12:22,764 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=9fe0640122ec,42593,1733947942024
2024-12-11T20:12:22,766 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9fe0640122ec,42593,1733947942024, state=OPENING
2024-12-11T20:12:22,768 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-11T20:12:22,770 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42593-0x100caf7f5180001, quorum=127.0.0.1:56424, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:12:22,770 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38011-0x100caf7f5180000, quorum=127.0.0.1:56424, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:12:22,771 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-11T20:12:22,771 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=9fe0640122ec,42593,1733947942024}]
2024-12-11T20:12:22,771 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-11T20:12:22,771 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-11T20:12:22,841 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-11T20:12:22,843 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:12:22,861 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:12:22,864 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:12:22,865 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:12:22,926 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-11T20:12:22,928 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59913, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-11T20:12:22,933 INFO  [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-12-11T20:12:22,933 INFO  [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-11T20:12:22,935 INFO  [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9fe0640122ec%2C42593%2C1733947942024.meta, suffix=.meta, logDir=hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024, archiveDir=hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/oldWALs, maxLogs=32
2024-12-11T20:12:22,936 INFO  [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
2024-12-11T20:12:22,941 INFO  [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
2024-12-11T20:12:22,942 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45941:45941),(127.0.0.1/127.0.0.1:35449:35449)]
2024-12-11T20:12:22,943 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-11T20:12:22,944 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-11T20:12:22,944 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-11T20:12:22,944 INFO  [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-11T20:12:22,944 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-11T20:12:22,944 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T20:12:22,944 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-12-11T20:12:22,944 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-12-11T20:12:22,946 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-11T20:12:22,948 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-11T20:12:22,948 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:12:22,948 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T20:12:22,949 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-11T20:12:22,950 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-11T20:12:22,950 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:12:22,950 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T20:12:22,951 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-11T20:12:22,951 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-11T20:12:22,951 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:12:22,952 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T20:12:22,952 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-11T20:12:22,995 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-11T20:12:22,996 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:12:22,997 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T20:12:22,997 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-11T20:12:22,998 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/hbase/meta/1588230740
2024-12-11T20:12:23,000 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/hbase/meta/1588230740
2024-12-11T20:12:23,001 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-11T20:12:23,001 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-11T20:12:23,002 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-11T20:12:23,004 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-11T20:12:23,005 INFO  [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=813485, jitterRate=0.03440055251121521}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-11T20:12:23,005 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-12-11T20:12:23,006 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1733947942945Writing region info on filesystem at 1733947942945Initializing all the Stores at 1733947942946 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733947942946Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733947942946Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733947942946Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733947942946Cleaning up temporary data from old regions at 1733947943001 (+55 ms)Running coprocessor post-open hooks at 1733947943005 (+4 ms)Region opened successfully at 1733947943006 (+1 ms)
2024-12-11T20:12:23,007 INFO  [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733947942925
2024-12-11T20:12:23,011 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-11T20:12:23,011 INFO  [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-12-11T20:12:23,013 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=9fe0640122ec,42593,1733947942024
2024-12-11T20:12:23,014 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9fe0640122ec,42593,1733947942024, state=OPEN
2024-12-11T20:12:23,018 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42593-0x100caf7f5180001, quorum=127.0.0.1:56424, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-11T20:12:23,018 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38011-0x100caf7f5180000, quorum=127.0.0.1:56424, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-11T20:12:23,018 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=9fe0640122ec,42593,1733947942024
2024-12-11T20:12:23,018 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-11T20:12:23,018 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-11T20:12:23,023 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-12-11T20:12:23,023 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9fe0640122ec,42593,1733947942024 in 247 msec
2024-12-11T20:12:23,026 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-12-11T20:12:23,026 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 663 msec
2024-12-11T20:12:23,027 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-11T20:12:23,027 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-12-11T20:12:23,029 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-11T20:12:23,029 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9fe0640122ec,42593,1733947942024, seqNum=-1]
2024-12-11T20:12:23,030 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-11T20:12:23,031 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53399, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-11T20:12:23,039 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 769 msec
2024-12-11T20:12:23,039 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733947943039, completionTime=-1
2024-12-11T20:12:23,039 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-11T20:12:23,040 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-12-11T20:12:23,042 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-12-11T20:12:23,042 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733948003042
2024-12-11T20:12:23,042 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733948063042
2024-12-11T20:12:23,042 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec
2024-12-11T20:12:23,042 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9fe0640122ec,38011,1733947941956-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-11T20:12:23,043 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9fe0640122ec,38011,1733947941956-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T20:12:23,043 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9fe0640122ec,38011,1733947941956-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T20:12:23,043 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-9fe0640122ec:38011, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T20:12:23,043 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-11T20:12:23,043 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-12-11T20:12:23,046 DEBUG [master/9fe0640122ec:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-12-11T20:12:23,048 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.985sec
2024-12-11T20:12:23,048 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-11T20:12:23,048 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-11T20:12:23,048 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-11T20:12:23,049 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-11T20:12:23,049 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-11T20:12:23,049 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9fe0640122ec,38011,1733947941956-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-11T20:12:23,049 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9fe0640122ec,38011,1733947941956-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-11T20:12:23,052 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-12-11T20:12:23,052 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-11T20:12:23,052 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9fe0640122ec,38011,1733947941956-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T20:12:23,149 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67c279b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-11T20:12:23,150 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9fe0640122ec,38011,-1 for getting cluster id
2024-12-11T20:12:23,150 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-12-11T20:12:23,152 DEBUG [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c629bce1-e155-4122-ab22-c357595bb826'
2024-12-11T20:12:23,152 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-12-11T20:12:23,152 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c629bce1-e155-4122-ab22-c357595bb826"

2024-12-11T20:12:23,153 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ce378b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-11T20:12:23,153 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9fe0640122ec,38011,-1]
2024-12-11T20:12:23,153 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-12-11T20:12:23,153 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T20:12:23,155 INFO  [HMaster-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36800, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-12-11T20:12:23,155 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@565a7a6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-11T20:12:23,156 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-11T20:12:23,157 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9fe0640122ec,42593,1733947942024, seqNum=-1]
2024-12-11T20:12:23,157 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-11T20:12:23,159 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45538, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-11T20:12:23,160 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=9fe0640122ec,38011,1733947941956
2024-12-11T20:12:23,161 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T20:12:23,163 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-12-11T20:12:23,179 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/9fe0640122ec:0 server-side Connection retries=45
2024-12-11T20:12:23,179 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-11T20:12:23,179 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-11T20:12:23,179 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-11T20:12:23,179 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-11T20:12:23,179 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-11T20:12:23,180 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-11T20:12:23,180 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-11T20:12:23,180 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42419
2024-12-11T20:12:23,182 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42419 connecting to ZooKeeper ensemble=127.0.0.1:56424
2024-12-11T20:12:23,183 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T20:12:23,186 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T20:12:23,192 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:424190x0, quorum=127.0.0.1:56424, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-11T20:12:23,192 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:424190x0, quorum=127.0.0.1:56424, baseZNode=/hbase Set watcher on existing znode=/hbase/running
2024-12-11T20:12:23,193 DEBUG [pool-381-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT
2024-12-11T20:12:23,194 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42419-0x100caf7f5180002 connected
2024-12-11T20:12:23,194 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-11T20:12:23,197 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-11T20:12:23,198 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:42419-0x100caf7f5180002, quorum=127.0.0.1:56424, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-11T20:12:23,199 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42419-0x100caf7f5180002, quorum=127.0.0.1:56424, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-11T20:12:23,200 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42419
2024-12-11T20:12:23,203 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42419
2024-12-11T20:12:23,205 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42419
2024-12-11T20:12:23,206 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42419
2024-12-11T20:12:23,207 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42419
2024-12-11T20:12:23,209 INFO  [RS:1;9fe0640122ec:42419 {}] regionserver.HRegionServer(746): ClusterId : c629bce1-e155-4122-ab22-c357595bb826
2024-12-11T20:12:23,209 DEBUG [RS:1;9fe0640122ec:42419 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-11T20:12:23,211 DEBUG [RS:1;9fe0640122ec:42419 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-11T20:12:23,211 DEBUG [RS:1;9fe0640122ec:42419 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-11T20:12:23,213 DEBUG [RS:1;9fe0640122ec:42419 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-11T20:12:23,214 DEBUG [RS:1;9fe0640122ec:42419 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9307741, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9fe0640122ec/172.17.0.2:0
2024-12-11T20:12:23,228 DEBUG [RS:1;9fe0640122ec:42419 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;9fe0640122ec:42419
2024-12-11T20:12:23,228 INFO  [RS:1;9fe0640122ec:42419 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-11T20:12:23,228 INFO  [RS:1;9fe0640122ec:42419 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-11T20:12:23,228 DEBUG [RS:1;9fe0640122ec:42419 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-11T20:12:23,229 INFO  [RS:1;9fe0640122ec:42419 {}] regionserver.HRegionServer(2659): reportForDuty to master=9fe0640122ec,38011,1733947941956 with port=42419, startcode=1733947943179
2024-12-11T20:12:23,229 DEBUG [RS:1;9fe0640122ec:42419 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-11T20:12:23,231 INFO  [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32983, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.3 (auth:SIMPLE), service=RegionServerStatusService
2024-12-11T20:12:23,231 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38011 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9fe0640122ec,42419,1733947943179
2024-12-11T20:12:23,231 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38011 {}] master.ServerManager(517): Registering regionserver=9fe0640122ec,42419,1733947943179
2024-12-11T20:12:23,233 DEBUG [RS:1;9fe0640122ec:42419 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f
2024-12-11T20:12:23,233 DEBUG [RS:1;9fe0640122ec:42419 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40619
2024-12-11T20:12:23,233 DEBUG [RS:1;9fe0640122ec:42419 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-11T20:12:23,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38011-0x100caf7f5180000, quorum=127.0.0.1:56424, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-11T20:12:23,236 DEBUG [RS:1;9fe0640122ec:42419 {}] zookeeper.ZKUtil(111): regionserver:42419-0x100caf7f5180002, quorum=127.0.0.1:56424, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9fe0640122ec,42419,1733947943179
2024-12-11T20:12:23,236 WARN  [RS:1;9fe0640122ec:42419 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-11T20:12:23,236 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9fe0640122ec,42419,1733947943179]
2024-12-11T20:12:23,236 INFO  [RS:1;9fe0640122ec:42419 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-11T20:12:23,236 DEBUG [RS:1;9fe0640122ec:42419 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179
2024-12-11T20:12:23,244 INFO  [RS:1;9fe0640122ec:42419 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-11T20:12:23,246 INFO  [RS:1;9fe0640122ec:42419 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-11T20:12:23,247 INFO  [RS:1;9fe0640122ec:42419 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-11T20:12:23,247 INFO  [RS:1;9fe0640122ec:42419 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-11T20:12:23,247 INFO  [RS:1;9fe0640122ec:42419 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-11T20:12:23,248 INFO  [RS:1;9fe0640122ec:42419 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-11T20:12:23,248 INFO  [RS:1;9fe0640122ec:42419 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-11T20:12:23,248 DEBUG [RS:1;9fe0640122ec:42419 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:12:23,248 DEBUG [RS:1;9fe0640122ec:42419 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:12:23,249 DEBUG [RS:1;9fe0640122ec:42419 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:12:23,249 DEBUG [RS:1;9fe0640122ec:42419 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:12:23,249 DEBUG [RS:1;9fe0640122ec:42419 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:12:23,249 DEBUG [RS:1;9fe0640122ec:42419 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9fe0640122ec:0, corePoolSize=2, maxPoolSize=2
2024-12-11T20:12:23,249 DEBUG [RS:1;9fe0640122ec:42419 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:12:23,249 DEBUG [RS:1;9fe0640122ec:42419 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:12:23,249 DEBUG [RS:1;9fe0640122ec:42419 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:12:23,249 DEBUG [RS:1;9fe0640122ec:42419 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:12:23,249 DEBUG [RS:1;9fe0640122ec:42419 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:12:23,249 DEBUG [RS:1;9fe0640122ec:42419 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:12:23,249 DEBUG [RS:1;9fe0640122ec:42419 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9fe0640122ec:0, corePoolSize=3, maxPoolSize=3
2024-12-11T20:12:23,249 DEBUG [RS:1;9fe0640122ec:42419 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9fe0640122ec:0, corePoolSize=3, maxPoolSize=3
2024-12-11T20:12:23,251 INFO  [RS:1;9fe0640122ec:42419 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-11T20:12:23,251 INFO  [RS:1;9fe0640122ec:42419 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-11T20:12:23,251 INFO  [RS:1;9fe0640122ec:42419 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-11T20:12:23,251 INFO  [RS:1;9fe0640122ec:42419 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-11T20:12:23,251 INFO  [RS:1;9fe0640122ec:42419 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-11T20:12:23,251 INFO  [RS:1;9fe0640122ec:42419 {}] hbase.ChoreService(168): Chore ScheduledChore name=9fe0640122ec,42419,1733947943179-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-11T20:12:23,275 INFO  [RS:1;9fe0640122ec:42419 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-11T20:12:23,276 INFO  [RS:1;9fe0640122ec:42419 {}] hbase.ChoreService(168): Chore ScheduledChore name=9fe0640122ec,42419,1733947943179-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-11T20:12:23,276 INFO  [RS:1;9fe0640122ec:42419 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T20:12:23,276 INFO  [RS:1;9fe0640122ec:42419 {}] regionserver.Replication(171): 9fe0640122ec,42419,1733947943179 started
2024-12-11T20:12:23,297 INFO  [RS:1;9fe0640122ec:42419 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T20:12:23,297 INFO  [RS:1;9fe0640122ec:42419 {}] regionserver.HRegionServer(1482): Serving as 9fe0640122ec,42419,1733947943179, RpcServer on 9fe0640122ec/172.17.0.2:42419, sessionid=0x100caf7f5180002
2024-12-11T20:12:23,297 DEBUG [RS:1;9fe0640122ec:42419 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-11T20:12:23,297 DEBUG [RS:1;9fe0640122ec:42419 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9fe0640122ec,42419,1733947943179
2024-12-11T20:12:23,297 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(2882): Started new server=Thread[RS:1;9fe0640122ec:42419,5,FailOnTimeoutGroup]
2024-12-11T20:12:23,297 DEBUG [RS:1;9fe0640122ec:42419 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9fe0640122ec,42419,1733947943179'
2024-12-11T20:12:23,297 DEBUG [RS:1;9fe0640122ec:42419 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-11T20:12:23,297 INFO  [Time-limited test {}] wal.TestLogRolling(207): Replication=2
2024-12-11T20:12:23,298 DEBUG [RS:1;9fe0640122ec:42419 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-11T20:12:23,298 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-12-11T20:12:23,298 DEBUG [RS:1;9fe0640122ec:42419 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-11T20:12:23,298 DEBUG [RS:1;9fe0640122ec:42419 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-11T20:12:23,298 DEBUG [RS:1;9fe0640122ec:42419 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9fe0640122ec,42419,1733947943179
2024-12-11T20:12:23,298 DEBUG [RS:1;9fe0640122ec:42419 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9fe0640122ec,42419,1733947943179'
2024-12-11T20:12:23,298 DEBUG [RS:1;9fe0640122ec:42419 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-11T20:12:23,299 DEBUG [RS:1;9fe0640122ec:42419 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-11T20:12:23,299 DEBUG [RS:1;9fe0640122ec:42419 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-11T20:12:23,299 INFO  [RS:1;9fe0640122ec:42419 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-11T20:12:23,299 INFO  [RS:1;9fe0640122ec:42419 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-11T20:12:23,299 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncConnectionImpl(321): The fetched master address is 9fe0640122ec,38011,1733947941956
2024-12-11T20:12:23,299 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3f6920e4
2024-12-11T20:12:23,300 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-12-11T20:12:23,302 INFO  [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36814, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-12-11T20:12:23,302 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38011 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-12-11T20:12:23,302 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38011 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-12-11T20:12:23,303 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38011 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRollOnDatanodeDeath', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-11T20:12:23,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38011 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath
2024-12-11T20:12:23,306 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_PRE_OPERATION
2024-12-11T20:12:23,306 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:12:23,306 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38011 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRollOnDatanodeDeath"
 procId is: 4
2024-12-11T20:12:23,307 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-11T20:12:23,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38011 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-11T20:12:23,315 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34893 is added to blk_1073741835_1011 (size=393)
2024-12-11T20:12:23,315 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36207 is added to blk_1073741835_1011 (size=393)
2024-12-11T20:12:23,317 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5109ba8daed9e21fd7c045352880c060, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1733947943302.5109ba8daed9e21fd7c045352880c060.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnDatanodeDeath', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f
2024-12-11T20:12:23,326 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34893 is added to blk_1073741836_1012 (size=76)
2024-12-11T20:12:23,327 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36207 is added to blk_1073741836_1012 (size=76)
2024-12-11T20:12:23,327 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1733947943302.5109ba8daed9e21fd7c045352880c060.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T20:12:23,327 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1722): Closing 5109ba8daed9e21fd7c045352880c060, disabling compactions & flushes
2024-12-11T20:12:23,327 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1733947943302.5109ba8daed9e21fd7c045352880c060.
2024-12-11T20:12:23,327 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1733947943302.5109ba8daed9e21fd7c045352880c060.
2024-12-11T20:12:23,327 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1733947943302.5109ba8daed9e21fd7c045352880c060. after waiting 0 ms
2024-12-11T20:12:23,327 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1733947943302.5109ba8daed9e21fd7c045352880c060.
2024-12-11T20:12:23,327 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1733947943302.5109ba8daed9e21fd7c045352880c060.
2024-12-11T20:12:23,327 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5109ba8daed9e21fd7c045352880c060:
Waiting for close lock at 1733947943327Disabling compacts and flushes for region at 1733947943327Disabling writes for close at 1733947943327Writing region close event to WAL at 1733947943327Closed at 1733947943327
2024-12-11T20:12:23,329 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ADD_TO_META
2024-12-11T20:12:23,330 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnDatanodeDeath,,1733947943302.5109ba8daed9e21fd7c045352880c060.","families":{"info":[{"qualifier":"regioninfo","vlen":75,"tag":[],"timestamp":"1733947943329"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733947943329"}]},"ts":"1733947943329"}
2024-12-11T20:12:23,333 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-12-11T20:12:23,334 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-11T20:12:23,335 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733947943334"}]},"ts":"1733947943334"}
2024-12-11T20:12:23,337 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLING in hbase:meta
2024-12-11T20:12:23,338 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=5109ba8daed9e21fd7c045352880c060, ASSIGN}]
2024-12-11T20:12:23,339 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=5109ba8daed9e21fd7c045352880c060, ASSIGN
2024-12-11T20:12:23,341 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=5109ba8daed9e21fd7c045352880c060, ASSIGN; state=OFFLINE, location=9fe0640122ec,42593,1733947942024; forceNewPlan=false, retain=false
2024-12-11T20:12:23,402 INFO  [RS:1;9fe0640122ec:42419 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9fe0640122ec%2C42419%2C1733947943179, suffix=, logDir=hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179, archiveDir=hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/oldWALs, maxLogs=32
2024-12-11T20:12:23,403 INFO  [RS:1;9fe0640122ec:42419 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9fe0640122ec%2C42419%2C1733947943179.1733947943403
2024-12-11T20:12:23,411 INFO  [RS:1;9fe0640122ec:42419 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
2024-12-11T20:12:23,412 DEBUG [RS:1;9fe0640122ec:42419 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45941:45941),(127.0.0.1/127.0.0.1:35449:35449)]
2024-12-11T20:12:23,492 INFO  [9fe0640122ec:38011 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 
2024-12-11T20:12:23,492 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=5109ba8daed9e21fd7c045352880c060, regionState=OPENING, regionLocation=9fe0640122ec,42593,1733947942024
2024-12-11T20:12:23,495 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=5109ba8daed9e21fd7c045352880c060, ASSIGN because future has completed
2024-12-11T20:12:23,496 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5109ba8daed9e21fd7c045352880c060, server=9fe0640122ec,42593,1733947942024}]
2024-12-11T20:12:23,655 INFO  [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRollOnDatanodeDeath,,1733947943302.5109ba8daed9e21fd7c045352880c060.
2024-12-11T20:12:23,656 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 5109ba8daed9e21fd7c045352880c060, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1733947943302.5109ba8daed9e21fd7c045352880c060.', STARTKEY => '', ENDKEY => ''}
2024-12-11T20:12:23,656 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnDatanodeDeath 5109ba8daed9e21fd7c045352880c060
2024-12-11T20:12:23,656 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1733947943302.5109ba8daed9e21fd7c045352880c060.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T20:12:23,656 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 5109ba8daed9e21fd7c045352880c060
2024-12-11T20:12:23,656 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 5109ba8daed9e21fd7c045352880c060
2024-12-11T20:12:23,658 INFO  [StoreOpener-5109ba8daed9e21fd7c045352880c060-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 5109ba8daed9e21fd7c045352880c060 
2024-12-11T20:12:23,660 INFO  [StoreOpener-5109ba8daed9e21fd7c045352880c060-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5109ba8daed9e21fd7c045352880c060 columnFamilyName info
2024-12-11T20:12:23,660 DEBUG [StoreOpener-5109ba8daed9e21fd7c045352880c060-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:12:23,660 INFO  [StoreOpener-5109ba8daed9e21fd7c045352880c060-1 {}] regionserver.HStore(327): Store=5109ba8daed9e21fd7c045352880c060/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-11T20:12:23,660 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 5109ba8daed9e21fd7c045352880c060
2024-12-11T20:12:23,661 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060
2024-12-11T20:12:23,662 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060
2024-12-11T20:12:23,662 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 5109ba8daed9e21fd7c045352880c060
2024-12-11T20:12:23,663 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 5109ba8daed9e21fd7c045352880c060
2024-12-11T20:12:23,664 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 5109ba8daed9e21fd7c045352880c060
2024-12-11T20:12:23,667 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-11T20:12:23,667 INFO  [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 5109ba8daed9e21fd7c045352880c060; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=768810, jitterRate=-0.02240772545337677}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-11T20:12:23,667 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5109ba8daed9e21fd7c045352880c060
2024-12-11T20:12:23,668 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 5109ba8daed9e21fd7c045352880c060:
Running coprocessor pre-open hook at 1733947943657Writing region info on filesystem at 1733947943657Initializing all the Stores at 1733947943657Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733947943657Cleaning up temporary data from old regions at 1733947943663 (+6 ms)Running coprocessor post-open hooks at 1733947943667 (+4 ms)Region opened successfully at 1733947943668 (+1 ms)
2024-12-11T20:12:23,670 INFO  [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRollOnDatanodeDeath,,1733947943302.5109ba8daed9e21fd7c045352880c060., pid=6, masterSystemTime=1733947943650
2024-12-11T20:12:23,673 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRollOnDatanodeDeath,,1733947943302.5109ba8daed9e21fd7c045352880c060.
2024-12-11T20:12:23,673 INFO  [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRollOnDatanodeDeath,,1733947943302.5109ba8daed9e21fd7c045352880c060.
2024-12-11T20:12:23,674 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=5109ba8daed9e21fd7c045352880c060, regionState=OPEN, openSeqNum=2, regionLocation=9fe0640122ec,42593,1733947942024
2024-12-11T20:12:23,676 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5109ba8daed9e21fd7c045352880c060, server=9fe0640122ec,42593,1733947942024 because future has completed
2024-12-11T20:12:23,681 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-12-11T20:12:23,681 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 5109ba8daed9e21fd7c045352880c060, server=9fe0640122ec,42593,1733947942024 in 182 msec
2024-12-11T20:12:23,684 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-12-11T20:12:23,684 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=5109ba8daed9e21fd7c045352880c060, ASSIGN in 343 msec
2024-12-11T20:12:23,685 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-11T20:12:23,685 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733947943685"}]},"ts":"1733947943685"}
2024-12-11T20:12:23,688 INFO  [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLED in hbase:meta
2024-12-11T20:12:23,689 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_POST_OPERATION
2024-12-11T20:12:23,692 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath in 386 msec
2024-12-11T20:12:28,446 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-11T20:12:28,449 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:12:28,467 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:12:28,470 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:12:28,470 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:12:28,485 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnDatanodeDeath'
2024-12-11T20:12:32,156 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-12-11T20:12:32,156 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-12-11T20:12:32,157 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath
2024-12-11T20:12:32,157 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath Metrics about Tables on a single HBase RegionServer
2024-12-11T20:12:32,157 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-11T20:12:32,157 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-12-11T20:12:32,157 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-12-11T20:12:32,157 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-12-11T20:12:33,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38011 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-11T20:12:33,378 INFO  [RPCClient-NioEventLoopGroup-4-11 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnDatanodeDeath completed
2024-12-11T20:12:33,378 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRollOnDatanodeDeath,, stopping at row=TestLogRolling-testLogRollOnDatanodeDeath ,, for max=2147483647 with caching=100
2024-12-11T20:12:33,381 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRollOnDatanodeDeath
2024-12-11T20:12:33,381 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRollOnDatanodeDeath,,1733947943302.5109ba8daed9e21fd7c045352880c060.
2024-12-11T20:12:33,394 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T20:12:33,399 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-11T20:12:33,400 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-11T20:12:33,401 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-11T20:12:33,401 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-11T20:12:33,401 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@20e1b523{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/hadoop.log.dir/,AVAILABLE}
2024-12-11T20:12:33,402 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2021586{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-11T20:12:33,521 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4e14c7af{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/java.io.tmpdir/jetty-localhost-42115-hadoop-hdfs-3_4_1-tests_jar-_-any-15784866168352706326/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T20:12:33,522 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5ae74a05{HTTP/1.1, (http/1.1)}{localhost:42115}
2024-12-11T20:12:33,522 INFO  [Time-limited test {}] server.Server(415): Started @117118ms
2024-12-11T20:12:33,523 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-11T20:12:33,561 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T20:12:33,564 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-11T20:12:33,565 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-11T20:12:33,565 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-11T20:12:33,565 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-11T20:12:33,566 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@32dac098{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/hadoop.log.dir/,AVAILABLE}
2024-12-11T20:12:33,566 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7f7383da{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-11T20:12:33,613 WARN  [Thread-832 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/cluster_f21598ea-49df-c0f8-b759-6a4f22597d28/data/data6/current/BP-773969328-172.17.0.2-1733947940734/current, will proceed with Du for space computation calculation, 
2024-12-11T20:12:33,613 WARN  [Thread-831 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/cluster_f21598ea-49df-c0f8-b759-6a4f22597d28/data/data5/current/BP-773969328-172.17.0.2-1733947940734/current, will proceed with Du for space computation calculation, 
2024-12-11T20:12:33,638 WARN  [Thread-811 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-11T20:12:33,641 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x45c06ac9f2dc1000 with lease ID 0x66af004492a15ccb: Processing first storage report for DS-20198564-3dc6-402d-9d4b-7023e3d1e57b from datanode DatanodeRegistration(127.0.0.1:35127, datanodeUuid=097d055e-02bc-41e5-87d2-025b15bf701d, infoPort=44277, infoSecurePort=0, ipcPort=45503, storageInfo=lv=-57;cid=testClusterID;nsid=1349632812;c=1733947940734)
2024-12-11T20:12:33,641 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x45c06ac9f2dc1000 with lease ID 0x66af004492a15ccb: from storage DS-20198564-3dc6-402d-9d4b-7023e3d1e57b node DatanodeRegistration(127.0.0.1:35127, datanodeUuid=097d055e-02bc-41e5-87d2-025b15bf701d, infoPort=44277, infoSecurePort=0, ipcPort=45503, storageInfo=lv=-57;cid=testClusterID;nsid=1349632812;c=1733947940734), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T20:12:33,641 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x45c06ac9f2dc1000 with lease ID 0x66af004492a15ccb: Processing first storage report for DS-4b95707e-bfb0-4720-ac66-99de6f6f6a33 from datanode DatanodeRegistration(127.0.0.1:35127, datanodeUuid=097d055e-02bc-41e5-87d2-025b15bf701d, infoPort=44277, infoSecurePort=0, ipcPort=45503, storageInfo=lv=-57;cid=testClusterID;nsid=1349632812;c=1733947940734)
2024-12-11T20:12:33,641 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x45c06ac9f2dc1000 with lease ID 0x66af004492a15ccb: from storage DS-4b95707e-bfb0-4720-ac66-99de6f6f6a33 node DatanodeRegistration(127.0.0.1:35127, datanodeUuid=097d055e-02bc-41e5-87d2-025b15bf701d, infoPort=44277, infoSecurePort=0, ipcPort=45503, storageInfo=lv=-57;cid=testClusterID;nsid=1349632812;c=1733947940734), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T20:12:33,690 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@44b90fe6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/java.io.tmpdir/jetty-localhost-40963-hadoop-hdfs-3_4_1-tests_jar-_-any-4422434954971265446/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T20:12:33,691 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@322568ee{HTTP/1.1, (http/1.1)}{localhost:40963}
2024-12-11T20:12:33,691 INFO  [Time-limited test {}] server.Server(415): Started @117286ms
2024-12-11T20:12:33,692 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-11T20:12:33,727 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T20:12:33,730 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-11T20:12:33,730 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-11T20:12:33,730 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-11T20:12:33,730 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-11T20:12:33,731 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7f55aa3b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/hadoop.log.dir/,AVAILABLE}
2024-12-11T20:12:33,731 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3fd17220{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-11T20:12:33,781 WARN  [Thread-866 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/cluster_f21598ea-49df-c0f8-b759-6a4f22597d28/data/data7/current/BP-773969328-172.17.0.2-1733947940734/current, will proceed with Du for space computation calculation, 
2024-12-11T20:12:33,781 WARN  [Thread-867 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/cluster_f21598ea-49df-c0f8-b759-6a4f22597d28/data/data8/current/BP-773969328-172.17.0.2-1733947940734/current, will proceed with Du for space computation calculation, 
2024-12-11T20:12:33,802 WARN  [Thread-846 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-11T20:12:33,804 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb129833d03b23f67 with lease ID 0x66af004492a15ccc: Processing first storage report for DS-cb434015-7991-4b0a-b64b-1a5c770c280f from datanode DatanodeRegistration(127.0.0.1:46049, datanodeUuid=d6172e83-42ec-48b7-a9de-ef79b33a691d, infoPort=42245, infoSecurePort=0, ipcPort=41991, storageInfo=lv=-57;cid=testClusterID;nsid=1349632812;c=1733947940734)
2024-12-11T20:12:33,804 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb129833d03b23f67 with lease ID 0x66af004492a15ccc: from storage DS-cb434015-7991-4b0a-b64b-1a5c770c280f node DatanodeRegistration(127.0.0.1:46049, datanodeUuid=d6172e83-42ec-48b7-a9de-ef79b33a691d, infoPort=42245, infoSecurePort=0, ipcPort=41991, storageInfo=lv=-57;cid=testClusterID;nsid=1349632812;c=1733947940734), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-11T20:12:33,804 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb129833d03b23f67 with lease ID 0x66af004492a15ccc: Processing first storage report for DS-8f9f2708-5c8d-4ab4-a5b2-d2a811809a64 from datanode DatanodeRegistration(127.0.0.1:46049, datanodeUuid=d6172e83-42ec-48b7-a9de-ef79b33a691d, infoPort=42245, infoSecurePort=0, ipcPort=41991, storageInfo=lv=-57;cid=testClusterID;nsid=1349632812;c=1733947940734)
2024-12-11T20:12:33,804 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb129833d03b23f67 with lease ID 0x66af004492a15ccc: from storage DS-8f9f2708-5c8d-4ab4-a5b2-d2a811809a64 node DatanodeRegistration(127.0.0.1:46049, datanodeUuid=d6172e83-42ec-48b7-a9de-ef79b33a691d, infoPort=42245, infoSecurePort=0, ipcPort=41991, storageInfo=lv=-57;cid=testClusterID;nsid=1349632812;c=1733947940734), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T20:12:33,850 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@a5ffcda{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/java.io.tmpdir/jetty-localhost-42877-hadoop-hdfs-3_4_1-tests_jar-_-any-4626147713438798760/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T20:12:33,851 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@61d45840{HTTP/1.1, (http/1.1)}{localhost:42877}
2024-12-11T20:12:33,851 INFO  [Time-limited test {}] server.Server(415): Started @117447ms
2024-12-11T20:12:33,852 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-11T20:12:33,949 WARN  [Thread-892 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/cluster_f21598ea-49df-c0f8-b759-6a4f22597d28/data/data9/current/BP-773969328-172.17.0.2-1733947940734/current, will proceed with Du for space computation calculation, 
2024-12-11T20:12:33,949 WARN  [Thread-893 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/cluster_f21598ea-49df-c0f8-b759-6a4f22597d28/data/data10/current/BP-773969328-172.17.0.2-1733947940734/current, will proceed with Du for space computation calculation, 
2024-12-11T20:12:33,968 WARN  [Thread-881 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-11T20:12:33,970 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe6b4757fb5d7df55 with lease ID 0x66af004492a15ccd: Processing first storage report for DS-b0a6421d-98c9-48af-ab16-a87f2dea8cbb from datanode DatanodeRegistration(127.0.0.1:38463, datanodeUuid=ce0d95cd-8c3d-4ad7-8b80-989a1cd7ec68, infoPort=41537, infoSecurePort=0, ipcPort=42607, storageInfo=lv=-57;cid=testClusterID;nsid=1349632812;c=1733947940734)
2024-12-11T20:12:33,970 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe6b4757fb5d7df55 with lease ID 0x66af004492a15ccd: from storage DS-b0a6421d-98c9-48af-ab16-a87f2dea8cbb node DatanodeRegistration(127.0.0.1:38463, datanodeUuid=ce0d95cd-8c3d-4ad7-8b80-989a1cd7ec68, infoPort=41537, infoSecurePort=0, ipcPort=42607, storageInfo=lv=-57;cid=testClusterID;nsid=1349632812;c=1733947940734), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-11T20:12:33,970 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe6b4757fb5d7df55 with lease ID 0x66af004492a15ccd: Processing first storage report for DS-464ccfa2-6532-448b-8a69-ef37db12187e from datanode DatanodeRegistration(127.0.0.1:38463, datanodeUuid=ce0d95cd-8c3d-4ad7-8b80-989a1cd7ec68, infoPort=41537, infoSecurePort=0, ipcPort=42607, storageInfo=lv=-57;cid=testClusterID;nsid=1349632812;c=1733947940734)
2024-12-11T20:12:33,970 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe6b4757fb5d7df55 with lease ID 0x66af004492a15ccd: from storage DS-464ccfa2-6532-448b-8a69-ef37db12187e node DatanodeRegistration(127.0.0.1:38463, datanodeUuid=ce0d95cd-8c3d-4ad7-8b80-989a1cd7ec68, infoPort=41537, infoSecurePort=0, ipcPort=42607, storageInfo=lv=-57;cid=testClusterID;nsid=1349632812;c=1733947940734), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T20:12:33,978 WARN  [ResponseProcessor for block BP-773969328-172.17.0.2-1733947940734:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-773969328-172.17.0.2-1733947940734:blk_1073741833_1009
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:33,978 WARN  [ResponseProcessor for block BP-773969328-172.17.0.2-1733947940734:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-773969328-172.17.0.2-1733947940734:blk_1073741830_1006
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:33,978 WARN  [ResponseProcessor for block BP-773969328-172.17.0.2-1733947940734:blk_1073741837_1013 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-773969328-172.17.0.2-1733947940734:blk_1073741837_1013
java.io.IOException: Bad response ERROR for BP-773969328-172.17.0.2-1733947940734:blk_1073741837_1013 from datanode DatanodeInfoWithStorage[127.0.0.1:36207,DS-a9e47326-b9f8-4a53-b473-bfa84b160cb2,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:33,978 WARN  [ResponseProcessor for block BP-773969328-172.17.0.2-1733947940734:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-773969328-172.17.0.2-1733947940734:blk_1073741834_1010
java.io.IOException: Bad response ERROR for BP-773969328-172.17.0.2-1733947940734:blk_1073741834_1010 from datanode DatanodeInfoWithStorage[127.0.0.1:36207,DS-a9e47326-b9f8-4a53-b473-bfa84b160cb2,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:33,978 WARN  [DataStreamer for file /user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/MasterData/WALs/9fe0640122ec,38011,1733947941956/9fe0640122ec%2C38011%2C1733947941956.1733947942154 block BP-773969328-172.17.0.2-1733947940734:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-773969328-172.17.0.2-1733947940734:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36207,DS-a9e47326-b9f8-4a53-b473-bfa84b160cb2,DISK], DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36207,DS-a9e47326-b9f8-4a53-b473-bfa84b160cb2,DISK]) is bad.
2024-12-11T20:12:33,978 WARN  [DataStreamer for file /user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.1733947942541 block BP-773969328-172.17.0.2-1733947940734:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-773969328-172.17.0.2-1733947940734:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36207,DS-a9e47326-b9f8-4a53-b473-bfa84b160cb2,DISK], DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36207,DS-a9e47326-b9f8-4a53-b473-bfa84b160cb2,DISK]) is bad.
2024-12-11T20:12:33,978 WARN  [DataStreamer for file /user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403 block BP-773969328-172.17.0.2-1733947940734:blk_1073741837_1013 {}] hdfs.DataStreamer(1731): Error Recovery for BP-773969328-172.17.0.2-1733947940734:blk_1073741837_1013 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK], DatanodeInfoWithStorage[127.0.0.1:36207,DS-a9e47326-b9f8-4a53-b473-bfa84b160cb2,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:36207,DS-a9e47326-b9f8-4a53-b473-bfa84b160cb2,DISK]) is bad.
2024-12-11T20:12:33,979 WARN  [DataStreamer for file /user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta block BP-773969328-172.17.0.2-1733947940734:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-773969328-172.17.0.2-1733947940734:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK], DatanodeInfoWithStorage[127.0.0.1:36207,DS-a9e47326-b9f8-4a53-b473-bfa84b160cb2,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:36207,DS-a9e47326-b9f8-4a53-b473-bfa84b160cb2,DISK]) is bad.
2024-12-11T20:12:33,979 WARN  [PacketResponder: BP-773969328-172.17.0.2-1733947940734:blk_1073741837_1013, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:36207] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:12:33,979 WARN  [PacketResponder: BP-773969328-172.17.0.2-1733947940734:blk_1073741834_1010, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:36207] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:12:33,980 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1278250962_22 at /127.0.0.1:42174 [Receiving block BP-773969328-172.17.0.2-1733947940734:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:34893:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:42174 dst: /127.0.0.1:34893
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:12:33,980 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2064808219_22 at /127.0.0.1:42194 [Receiving block BP-773969328-172.17.0.2-1733947940734:blk_1073741837_1013] {}] datanode.DataXceiver(331): 127.0.0.1:34893:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:42194 dst: /127.0.0.1:34893
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:12:33,980 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-611163082_22 at /127.0.0.1:42118 [Receiving block BP-773969328-172.17.0.2-1733947940734:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:34893:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:42118 dst: /127.0.0.1:34893
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:12:33,980 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-611163082_22 at /127.0.0.1:39818 [Receiving block BP-773969328-172.17.0.2-1733947940734:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:36207:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:39818 dst: /127.0.0.1:36207
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:12:33,980 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1278250962_22 at /127.0.0.1:39850 [Receiving block BP-773969328-172.17.0.2-1733947940734:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:36207:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:39850 dst: /127.0.0.1:36207
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:12:33,980 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1278250962_22 at /127.0.0.1:42160 [Receiving block BP-773969328-172.17.0.2-1733947940734:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:34893:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:42160 dst: /127.0.0.1:34893
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:12:33,980 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2064808219_22 at /127.0.0.1:39888 [Receiving block BP-773969328-172.17.0.2-1733947940734:blk_1073741837_1013] {}] datanode.DataXceiver(331): 127.0.0.1:36207:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:39888 dst: /127.0.0.1:36207
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:12:33,981 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6222f16{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T20:12:33,980 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1278250962_22 at /127.0.0.1:39860 [Receiving block BP-773969328-172.17.0.2-1733947940734:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:36207:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:39860 dst: /127.0.0.1:36207
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:12:33,982 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5d9b1613{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-11T20:12:33,982 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-11T20:12:33,982 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@20aa2ea7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-11T20:12:33,982 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3b918d2a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/hadoop.log.dir/,STOPPED}
2024-12-11T20:12:33,983 WARN  [BP-773969328-172.17.0.2-1733947940734 heartbeating to localhost/127.0.0.1:40619 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-11T20:12:33,983 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-11T20:12:33,983 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-11T20:12:33,983 WARN  [BP-773969328-172.17.0.2-1733947940734 heartbeating to localhost/127.0.0.1:40619 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-773969328-172.17.0.2-1733947940734 (Datanode Uuid 669f8eed-2f96-4854-8c9a-69c5c7a3d0c2) service to localhost/127.0.0.1:40619
2024-12-11T20:12:33,984 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/cluster_f21598ea-49df-c0f8-b759-6a4f22597d28/data/data3/current/BP-773969328-172.17.0.2-1733947940734 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T20:12:33,984 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/cluster_f21598ea-49df-c0f8-b759-6a4f22597d28/data/data4/current/BP-773969328-172.17.0.2-1733947940734 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T20:12:33,984 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-11T20:12:33,985 WARN  [DataStreamer for file /user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403 block BP-773969328-172.17.0.2-1733947940734:blk_1073741837_1013 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741837_1013
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Sender.send(Sender.java:84) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Sender.writeBlock(Sender.java:171) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1922) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:33,987 ERROR [org.apache.hadoop.hdfs.server.datanode.DataXceiver@715e6109 {}] datanode.DataXceiver(331): 127.0.0.1:34893:DataXceiver error processing unknown operation  src: /127.0.0.1:53346 dst: /127.0.0.1:34893
java.io.IOException: Server closed.
	at org.apache.hadoop.hdfs.server.datanode.DataXceiverServer.addPeer(DataXceiverServer.java:334) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:232) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:12:33,988 WARN  [DataStreamer for file /user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/MasterData/WALs/9fe0640122ec,38011,1733947941956/9fe0640122ec%2C38011%2C1733947941956.1733947942154 block BP-773969328-172.17.0.2-1733947940734:blk_1073741830_1006 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741830_1006
java.net.SocketException: Connection reset
	at sun.nio.ch.SocketChannelImpl.throwConnectionReset(SocketChannelImpl.java:394) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:426) ~[?:?]
	at org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118) ~[hadoop-common-3.4.1.jar:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:527) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1931) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:33,988 WARN  [DataStreamer for file /user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta block BP-773969328-172.17.0.2-1733947940734:blk_1073741834_1010 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741834_1010
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:33,988 WARN  [DataStreamer for file /user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.1733947942541 block BP-773969328-172.17.0.2-1733947940734:blk_1073741833_1009 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741833_1009
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:33,990 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3e26ba04{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T20:12:33,990 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7f515a91{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-11T20:12:33,990 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-11T20:12:33,990 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@ee6b493{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-11T20:12:33,990 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6787773a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/hadoop.log.dir/,STOPPED}
2024-12-11T20:12:33,992 WARN  [BP-773969328-172.17.0.2-1733947940734 heartbeating to localhost/127.0.0.1:40619 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-11T20:12:33,992 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-11T20:12:33,992 WARN  [BP-773969328-172.17.0.2-1733947940734 heartbeating to localhost/127.0.0.1:40619 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-773969328-172.17.0.2-1733947940734 (Datanode Uuid ab505b67-ff6c-4d4a-96e5-79bfe9f4b042) service to localhost/127.0.0.1:40619
2024-12-11T20:12:33,992 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-11T20:12:33,992 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/cluster_f21598ea-49df-c0f8-b759-6a4f22597d28/data/data1/current/BP-773969328-172.17.0.2-1733947940734 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T20:12:33,992 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/cluster_f21598ea-49df-c0f8-b759-6a4f22597d28/data/data2/current/BP-773969328-172.17.0.2-1733947940734 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T20:12:33,993 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-11T20:12:33,996 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRollOnDatanodeDeath', row='row0002', locateType=CURRENT is [region=TestLogRolling-testLogRollOnDatanodeDeath,,1733947943302.5109ba8daed9e21fd7c045352880c060., hostname=9fe0640122ec,42593,1733947942024, seqNum=2]
2024-12-11T20:12:33,998 ERROR [FSHLog-0-hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f-prefix:9fe0640122ec,42593,1733947942024 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:33,998 WARN  [FSHLog-0-hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f-prefix:9fe0640122ec,42593,1733947942024 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:33,998 INFO  [regionserver/9fe0640122ec:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:33,998 DEBUG [regionserver/9fe0640122ec:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 9fe0640122ec%2C42593%2C1733947942024:(num 1733947942541) roll requested
2024-12-11T20:12:33,999 INFO  [regionserver/9fe0640122ec:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9fe0640122ec%2C42593%2C1733947942024.1733947953999
2024-12-11T20:12:34,002 WARN  [Thread-904 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741838_1018
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:34,002 WARN  [Thread-904 {}] hdfs.DataStreamer(1731): Error Recovery for BP-773969328-172.17.0.2-1733947940734:blk_1073741838_1018 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK], DatanodeInfoWithStorage[127.0.0.1:35127,DS-20198564-3dc6-402d-9d4b-7023e3d1e57b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]) is bad.
2024-12-11T20:12:34,002 WARN  [Thread-904 {}] hdfs.DataStreamer(1850): Abandoning BP-773969328-172.17.0.2-1733947940734:blk_1073741838_1018
2024-12-11T20:12:34,004 WARN  [Thread-904 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]
2024-12-11T20:12:34,013 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:12:34,013 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:12:34,013 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:12:34,013 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:12:34,013 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:12:34,013 INFO  [regionserver/9fe0640122ec:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.1733947942541 with entries=1, filesize=455 B; new WAL /user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.1733947953999
2024-12-11T20:12:34,014 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:34,014 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:34,015 DEBUG [regionserver/9fe0640122ec:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44277:44277),(127.0.0.1/127.0.0.1:42245:42245)]
2024-12-11T20:12:34,015 DEBUG [regionserver/9fe0640122ec:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.1733947942541 is not closed yet, will try archiving it next time
2024-12-11T20:12:34,015 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils
2024-12-11T20:12:34,016 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease()
2024-12-11T20:12:34,016 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.1733947942541
2024-12-11T20:12:34,019 WARN  [IPC Server handler 2 on default port 40619 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.1733947942541 has not been closed. Lease recovery is in progress. RecoveryId = 1020 for block blk_1073741833_1009
2024-12-11T20:12:34,023 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.1733947942541 after 5ms
2024-12-11T20:12:34,117 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:35,251 INFO  [regionserver/9fe0640122ec:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:36,015 INFO  [regionserver/9fe0640122ec:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:36,017 INFO  [Time-limited test {}] wal.TestLogRolling(261): log.getCurrentFileName(): hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.1733947953999
2024-12-11T20:12:36,017 WARN  [ResponseProcessor for block BP-773969328-172.17.0.2-1733947940734:blk_1073741839_1019 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-773969328-172.17.0.2-1733947940734:blk_1073741839_1019
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:36,018 WARN  [DataStreamer for file /user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.1733947953999 block BP-773969328-172.17.0.2-1733947940734:blk_1073741839_1019 {}] hdfs.DataStreamer(1731): Error Recovery for BP-773969328-172.17.0.2-1733947940734:blk_1073741839_1019 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35127,DS-20198564-3dc6-402d-9d4b-7023e3d1e57b,DISK], DatanodeInfoWithStorage[127.0.0.1:46049,DS-cb434015-7991-4b0a-b64b-1a5c770c280f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35127,DS-20198564-3dc6-402d-9d4b-7023e3d1e57b,DISK]) is bad.
2024-12-11T20:12:36,018 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1278250962_22 at /127.0.0.1:45312 [Receiving block BP-773969328-172.17.0.2-1733947940734:blk_1073741839_1019] {}] datanode.DataXceiver(331): 127.0.0.1:35127:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:45312 dst: /127.0.0.1:35127
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:12:36,018 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1278250962_22 at /127.0.0.1:38268 [Receiving block BP-773969328-172.17.0.2-1733947940734:blk_1073741839_1019] {}] datanode.DataXceiver(331): 127.0.0.1:46049:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:38268 dst: /127.0.0.1:46049
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:12:36,020 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4e14c7af{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T20:12:36,020 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5ae74a05{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-11T20:12:36,021 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-11T20:12:36,021 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2021586{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-11T20:12:36,021 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@20e1b523{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/hadoop.log.dir/,STOPPED}
2024-12-11T20:12:36,022 WARN  [BP-773969328-172.17.0.2-1733947940734 heartbeating to localhost/127.0.0.1:40619 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-11T20:12:36,022 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-11T20:12:36,022 WARN  [BP-773969328-172.17.0.2-1733947940734 heartbeating to localhost/127.0.0.1:40619 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-773969328-172.17.0.2-1733947940734 (Datanode Uuid 097d055e-02bc-41e5-87d2-025b15bf701d) service to localhost/127.0.0.1:40619
2024-12-11T20:12:36,022 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-11T20:12:36,022 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/cluster_f21598ea-49df-c0f8-b759-6a4f22597d28/data/data5/current/BP-773969328-172.17.0.2-1733947940734 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T20:12:36,022 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/cluster_f21598ea-49df-c0f8-b759-6a4f22597d28/data/data6/current/BP-773969328-172.17.0.2-1733947940734 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T20:12:36,023 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-11T20:12:36,117 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:37,252 INFO  [regionserver/9fe0640122ec:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:38,016 WARN  [regionserver/9fe0640122ec:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46049,DS-cb434015-7991-4b0a-b64b-1a5c770c280f,DISK]]
2024-12-11T20:12:38,016 INFO  [regionserver/9fe0640122ec:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:38,016 DEBUG [regionserver/9fe0640122ec:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 9fe0640122ec%2C42593%2C1733947942024:(num 1733947953999) roll requested
2024-12-11T20:12:38,017 INFO  [regionserver/9fe0640122ec:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9fe0640122ec%2C42593%2C1733947942024.1733947958017
2024-12-11T20:12:38,020 WARN  [Thread-913 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741840_1022
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:38,020 WARN  [Thread-913 {}] hdfs.DataStreamer(1731): Error Recovery for BP-773969328-172.17.0.2-1733947940734:blk_1073741840_1022 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35127,DS-20198564-3dc6-402d-9d4b-7023e3d1e57b,DISK], DatanodeInfoWithStorage[127.0.0.1:46049,DS-cb434015-7991-4b0a-b64b-1a5c770c280f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35127,DS-20198564-3dc6-402d-9d4b-7023e3d1e57b,DISK]) is bad.
2024-12-11T20:12:38,020 WARN  [Thread-913 {}] hdfs.DataStreamer(1850): Abandoning BP-773969328-172.17.0.2-1733947940734:blk_1073741840_1022
2024-12-11T20:12:38,021 WARN  [Thread-913 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35127,DS-20198564-3dc6-402d-9d4b-7023e3d1e57b,DISK]
2024-12-11T20:12:38,024 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.1733947942541 after 4008ms
2024-12-11T20:12:38,025 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:12:38,025 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:12:38,025 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:12:38,025 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:12:38,025 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:12:38,026 INFO  [regionserver/9fe0640122ec:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.1733947953999 with entries=2, filesize=2.37 KB; new WAL /user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.1733947958017
2024-12-11T20:12:38,027 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46049 is added to blk_1073741839_1021 (size=2431)
2024-12-11T20:12:38,028 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-12-11T20:12:38,032 DEBUG [regionserver/9fe0640122ec:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42245:42245),(127.0.0.1/127.0.0.1:41537:41537)]
2024-12-11T20:12:38,032 DEBUG [regionserver/9fe0640122ec:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.1733947942541 is not closed yet, will try archiving it next time
2024-12-11T20:12:38,032 DEBUG [regionserver/9fe0640122ec:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.1733947953999 is not closed yet, will try archiving it next time
2024-12-11T20:12:38,118 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:38,429 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.1733947942541 is not closed yet, will try archiving it next time
2024-12-11T20:12:39,252 INFO  [regionserver/9fe0640122ec:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:39,817 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@19cf6a60[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:46049, datanodeUuid=d6172e83-42ec-48b7-a9de-ef79b33a691d, infoPort=42245, infoSecurePort=0, ipcPort=41991, storageInfo=lv=-57;cid=testClusterID;nsid=1349632812;c=1733947940734):Failed to transfer BP-773969328-172.17.0.2-1733947940734:blk_1073741839_1021 to 127.0.0.1:36207 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:12:40,033 INFO  [regionserver/9fe0640122ec:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:40,034 WARN  [ResponseProcessor for block BP-773969328-172.17.0.2-1733947940734:blk_1073741841_1023 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-773969328-172.17.0.2-1733947940734:blk_1073741841_1023
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:40,035 WARN  [DataStreamer for file /user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.1733947958017 block BP-773969328-172.17.0.2-1733947940734:blk_1073741841_1023 {}] hdfs.DataStreamer(1731): Error Recovery for BP-773969328-172.17.0.2-1733947940734:blk_1073741841_1023 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46049,DS-cb434015-7991-4b0a-b64b-1a5c770c280f,DISK], DatanodeInfoWithStorage[127.0.0.1:38463,DS-b0a6421d-98c9-48af-ab16-a87f2dea8cbb,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:46049,DS-cb434015-7991-4b0a-b64b-1a5c770c280f,DISK]) is bad.
2024-12-11T20:12:40,035 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1278250962_22 at /127.0.0.1:58784 [Receiving block BP-773969328-172.17.0.2-1733947940734:blk_1073741841_1023] {}] datanode.DataXceiver(331): 127.0.0.1:46049:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:58784 dst: /127.0.0.1:46049
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:12:40,035 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1278250962_22 at /127.0.0.1:40256 [Receiving block BP-773969328-172.17.0.2-1733947940734:blk_1073741841_1023] {}] datanode.DataXceiver(331): 127.0.0.1:38463:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:40256 dst: /127.0.0.1:38463
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:12:40,037 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@44b90fe6{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T20:12:40,037 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@322568ee{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-11T20:12:40,037 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-11T20:12:40,037 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7f7383da{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-11T20:12:40,037 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@32dac098{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/hadoop.log.dir/,STOPPED}
2024-12-11T20:12:40,039 WARN  [BP-773969328-172.17.0.2-1733947940734 heartbeating to localhost/127.0.0.1:40619 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-11T20:12:40,039 WARN  [BP-773969328-172.17.0.2-1733947940734 heartbeating to localhost/127.0.0.1:40619 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-773969328-172.17.0.2-1733947940734 (Datanode Uuid d6172e83-42ec-48b7-a9de-ef79b33a691d) service to localhost/127.0.0.1:40619
2024-12-11T20:12:40,039 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-11T20:12:40,039 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-11T20:12:40,040 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/cluster_f21598ea-49df-c0f8-b759-6a4f22597d28/data/data7/current/BP-773969328-172.17.0.2-1733947940734 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T20:12:40,040 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/cluster_f21598ea-49df-c0f8-b759-6a4f22597d28/data/data8/current/BP-773969328-172.17.0.2-1733947940734 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T20:12:40,040 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-11T20:12:40,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42593 {}] regionserver.HRegion(8855): Flush requested on 5109ba8daed9e21fd7c045352880c060
2024-12-11T20:12:40,049 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5109ba8daed9e21fd7c045352880c060 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-11T20:12:40,068 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/.tmp/info/c715426bd5aa4aa0af6e97e1bd20c1f1 is 1080, key is row0002/info:/1733947956025/Put/seqid=0
2024-12-11T20:12:40,070 WARN  [Thread-923 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741842_1025
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:40,071 WARN  [Thread-923 {}] hdfs.DataStreamer(1731): Error Recovery for BP-773969328-172.17.0.2-1733947940734:blk_1073741842_1025 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36207,DS-a9e47326-b9f8-4a53-b473-bfa84b160cb2,DISK], DatanodeInfoWithStorage[127.0.0.1:35127,DS-20198564-3dc6-402d-9d4b-7023e3d1e57b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36207,DS-a9e47326-b9f8-4a53-b473-bfa84b160cb2,DISK]) is bad.
2024-12-11T20:12:40,071 WARN  [Thread-923 {}] hdfs.DataStreamer(1850): Abandoning BP-773969328-172.17.0.2-1733947940734:blk_1073741842_1025
2024-12-11T20:12:40,071 WARN  [Thread-923 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36207,DS-a9e47326-b9f8-4a53-b473-bfa84b160cb2,DISK]
2024-12-11T20:12:40,072 WARN  [Thread-923 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741843_1026
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:40,073 WARN  [Thread-923 {}] hdfs.DataStreamer(1731): Error Recovery for BP-773969328-172.17.0.2-1733947940734:blk_1073741843_1026 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35127,DS-20198564-3dc6-402d-9d4b-7023e3d1e57b,DISK], DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35127,DS-20198564-3dc6-402d-9d4b-7023e3d1e57b,DISK]) is bad.
2024-12-11T20:12:40,073 WARN  [Thread-923 {}] hdfs.DataStreamer(1850): Abandoning BP-773969328-172.17.0.2-1733947940734:blk_1073741843_1026
2024-12-11T20:12:40,073 WARN  [Thread-923 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35127,DS-20198564-3dc6-402d-9d4b-7023e3d1e57b,DISK]
2024-12-11T20:12:40,075 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1278250962_22 at /127.0.0.1:40274 [Receiving block BP-773969328-172.17.0.2-1733947940734:blk_1073741844_1027] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/cluster_f21598ea-49df-c0f8-b759-6a4f22597d28/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/cluster_f21598ea-49df-c0f8-b759-6a4f22597d28/data/data10]'}, localName='127.0.0.1:38463', datanodeUuid='ce0d95cd-8c3d-4ad7-8b80-989a1cd7ec68', xmitsInProgress=0}:Exception transferring block BP-773969328-172.17.0.2-1733947940734:blk_1073741844_1027 to mirror 127.0.0.1:34893
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:12:40,075 WARN  [Thread-923 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741844_1027
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:34893
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:40,076 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1278250962_22 at /127.0.0.1:40274 [Receiving block BP-773969328-172.17.0.2-1733947940734:blk_1073741844_1027] {}] datanode.BlockReceiver(316): Block 1073741844 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-11T20:12:40,076 WARN  [Thread-923 {}] hdfs.DataStreamer(1731): Error Recovery for BP-773969328-172.17.0.2-1733947940734:blk_1073741844_1027 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38463,DS-b0a6421d-98c9-48af-ab16-a87f2dea8cbb,DISK], DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]) is bad.
2024-12-11T20:12:40,076 WARN  [Thread-923 {}] hdfs.DataStreamer(1850): Abandoning BP-773969328-172.17.0.2-1733947940734:blk_1073741844_1027
2024-12-11T20:12:40,076 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1278250962_22 at /127.0.0.1:40274 [Receiving block BP-773969328-172.17.0.2-1733947940734:blk_1073741844_1027] {}] datanode.DataXceiver(331): 127.0.0.1:38463:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:40274 dst: /127.0.0.1:38463
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:12:40,076 WARN  [Thread-923 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]
2024-12-11T20:12:40,078 WARN  [Thread-923 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741845_1028
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:46049
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:40,078 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1278250962_22 at /127.0.0.1:40288 [Receiving block BP-773969328-172.17.0.2-1733947940734:blk_1073741845_1028] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/cluster_f21598ea-49df-c0f8-b759-6a4f22597d28/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/cluster_f21598ea-49df-c0f8-b759-6a4f22597d28/data/data10]'}, localName='127.0.0.1:38463', datanodeUuid='ce0d95cd-8c3d-4ad7-8b80-989a1cd7ec68', xmitsInProgress=0}:Exception transferring block BP-773969328-172.17.0.2-1733947940734:blk_1073741845_1028 to mirror 127.0.0.1:46049
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:12:40,078 WARN  [Thread-923 {}] hdfs.DataStreamer(1731): Error Recovery for BP-773969328-172.17.0.2-1733947940734:blk_1073741845_1028 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38463,DS-b0a6421d-98c9-48af-ab16-a87f2dea8cbb,DISK], DatanodeInfoWithStorage[127.0.0.1:46049,DS-cb434015-7991-4b0a-b64b-1a5c770c280f,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:46049,DS-cb434015-7991-4b0a-b64b-1a5c770c280f,DISK]) is bad.
2024-12-11T20:12:40,078 WARN  [Thread-923 {}] hdfs.DataStreamer(1850): Abandoning BP-773969328-172.17.0.2-1733947940734:blk_1073741845_1028
2024-12-11T20:12:40,079 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1278250962_22 at /127.0.0.1:40288 [Receiving block BP-773969328-172.17.0.2-1733947940734:blk_1073741845_1028] {}] datanode.BlockReceiver(316): Block 1073741845 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-11T20:12:40,079 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1278250962_22 at /127.0.0.1:40288 [Receiving block BP-773969328-172.17.0.2-1733947940734:blk_1073741845_1028] {}] datanode.DataXceiver(331): 127.0.0.1:38463:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:40288 dst: /127.0.0.1:38463
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:12:40,079 WARN  [Thread-923 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46049,DS-cb434015-7991-4b0a-b64b-1a5c770c280f,DISK]
2024-12-11T20:12:40,080 WARN  [IPC Server handler 2 on default port 40619 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-11T20:12:40,080 WARN  [IPC Server handler 2 on default port 40619 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-11T20:12:40,080 WARN  [IPC Server handler 2 on default port 40619 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-11T20:12:40,083 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38463 is added to blk_1073741846_1029 (size=10347)
2024-12-11T20:12:40,118 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:40,484 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/.tmp/info/c715426bd5aa4aa0af6e97e1bd20c1f1
2024-12-11T20:12:40,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/.tmp/info/c715426bd5aa4aa0af6e97e1bd20c1f1 as hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/info/c715426bd5aa4aa0af6e97e1bd20c1f1
2024-12-11T20:12:40,499 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/info/c715426bd5aa4aa0af6e97e1bd20c1f1, entries=5, sequenceid=11, filesize=10.1 K
2024-12-11T20:12:40,500 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=9.45 KB/9681 for 5109ba8daed9e21fd7c045352880c060 in 451ms, sequenceid=11, compaction requested=false
2024-12-11T20:12:40,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5109ba8daed9e21fd7c045352880c060:

2024-12-11T20:12:40,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42593 {}] regionserver.HRegion(8855): Flush requested on 5109ba8daed9e21fd7c045352880c060
2024-12-11T20:12:40,671 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5109ba8daed9e21fd7c045352880c060 1/1 column families, dataSize=10.50 KB heapSize=11.50 KB
2024-12-11T20:12:40,678 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/.tmp/info/f4335984ea3a439692479308d3f53619 is 1080, key is row0007/info:/1733947960050/Put/seqid=0
2024-12-11T20:12:40,680 WARN  [Thread-930 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741847_1030
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:40,680 WARN  [Thread-930 {}] hdfs.DataStreamer(1731): Error Recovery for BP-773969328-172.17.0.2-1733947940734:blk_1073741847_1030 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35127,DS-20198564-3dc6-402d-9d4b-7023e3d1e57b,DISK], DatanodeInfoWithStorage[127.0.0.1:38463,DS-b0a6421d-98c9-48af-ab16-a87f2dea8cbb,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35127,DS-20198564-3dc6-402d-9d4b-7023e3d1e57b,DISK]) is bad.
2024-12-11T20:12:40,680 WARN  [Thread-930 {}] hdfs.DataStreamer(1850): Abandoning BP-773969328-172.17.0.2-1733947940734:blk_1073741847_1030
2024-12-11T20:12:40,681 WARN  [Thread-930 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35127,DS-20198564-3dc6-402d-9d4b-7023e3d1e57b,DISK]
2024-12-11T20:12:40,685 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1278250962_22 at /127.0.0.1:40322 [Receiving block BP-773969328-172.17.0.2-1733947940734:blk_1073741848_1031] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/cluster_f21598ea-49df-c0f8-b759-6a4f22597d28/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/cluster_f21598ea-49df-c0f8-b759-6a4f22597d28/data/data10]'}, localName='127.0.0.1:38463', datanodeUuid='ce0d95cd-8c3d-4ad7-8b80-989a1cd7ec68', xmitsInProgress=0}:Exception transferring block BP-773969328-172.17.0.2-1733947940734:blk_1073741848_1031 to mirror 127.0.0.1:34893
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:12:40,685 WARN  [Thread-930 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741848_1031
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:34893
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:40,685 WARN  [Thread-930 {}] hdfs.DataStreamer(1731): Error Recovery for BP-773969328-172.17.0.2-1733947940734:blk_1073741848_1031 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38463,DS-b0a6421d-98c9-48af-ab16-a87f2dea8cbb,DISK], DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]) is bad.
2024-12-11T20:12:40,685 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1278250962_22 at /127.0.0.1:40322 [Receiving block BP-773969328-172.17.0.2-1733947940734:blk_1073741848_1031] {}] datanode.BlockReceiver(316): Block 1073741848 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-11T20:12:40,685 WARN  [Thread-930 {}] hdfs.DataStreamer(1850): Abandoning BP-773969328-172.17.0.2-1733947940734:blk_1073741848_1031
2024-12-11T20:12:40,685 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1278250962_22 at /127.0.0.1:40322 [Receiving block BP-773969328-172.17.0.2-1733947940734:blk_1073741848_1031] {}] datanode.DataXceiver(331): 127.0.0.1:38463:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:40322 dst: /127.0.0.1:38463
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:12:40,686 WARN  [Thread-930 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]
2024-12-11T20:12:40,687 WARN  [Thread-930 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741849_1032
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:40,688 WARN  [Thread-930 {}] hdfs.DataStreamer(1731): Error Recovery for BP-773969328-172.17.0.2-1733947940734:blk_1073741849_1032 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46049,DS-cb434015-7991-4b0a-b64b-1a5c770c280f,DISK], DatanodeInfoWithStorage[127.0.0.1:36207,DS-a9e47326-b9f8-4a53-b473-bfa84b160cb2,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:46049,DS-cb434015-7991-4b0a-b64b-1a5c770c280f,DISK]) is bad.
2024-12-11T20:12:40,688 WARN  [Thread-930 {}] hdfs.DataStreamer(1850): Abandoning BP-773969328-172.17.0.2-1733947940734:blk_1073741849_1032
2024-12-11T20:12:40,688 WARN  [Thread-930 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46049,DS-cb434015-7991-4b0a-b64b-1a5c770c280f,DISK]
2024-12-11T20:12:40,691 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1278250962_22 at /127.0.0.1:40336 [Receiving block BP-773969328-172.17.0.2-1733947940734:blk_1073741850_1033] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/cluster_f21598ea-49df-c0f8-b759-6a4f22597d28/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/cluster_f21598ea-49df-c0f8-b759-6a4f22597d28/data/data10]'}, localName='127.0.0.1:38463', datanodeUuid='ce0d95cd-8c3d-4ad7-8b80-989a1cd7ec68', xmitsInProgress=0}:Exception transferring block BP-773969328-172.17.0.2-1733947940734:blk_1073741850_1033 to mirror 127.0.0.1:36207
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:12:40,691 WARN  [Thread-930 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741850_1033
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:36207
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:40,691 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1278250962_22 at /127.0.0.1:40336 [Receiving block BP-773969328-172.17.0.2-1733947940734:blk_1073741850_1033] {}] datanode.BlockReceiver(316): Block 1073741850 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-11T20:12:40,691 WARN  [Thread-930 {}] hdfs.DataStreamer(1731): Error Recovery for BP-773969328-172.17.0.2-1733947940734:blk_1073741850_1033 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38463,DS-b0a6421d-98c9-48af-ab16-a87f2dea8cbb,DISK], DatanodeInfoWithStorage[127.0.0.1:36207,DS-a9e47326-b9f8-4a53-b473-bfa84b160cb2,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:36207,DS-a9e47326-b9f8-4a53-b473-bfa84b160cb2,DISK]) is bad.
2024-12-11T20:12:40,691 WARN  [Thread-930 {}] hdfs.DataStreamer(1850): Abandoning BP-773969328-172.17.0.2-1733947940734:blk_1073741850_1033
2024-12-11T20:12:40,691 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1278250962_22 at /127.0.0.1:40336 [Receiving block BP-773969328-172.17.0.2-1733947940734:blk_1073741850_1033] {}] datanode.DataXceiver(331): 127.0.0.1:38463:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:40336 dst: /127.0.0.1:38463
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:12:40,693 WARN  [Thread-930 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36207,DS-a9e47326-b9f8-4a53-b473-bfa84b160cb2,DISK]
2024-12-11T20:12:40,694 WARN  [IPC Server handler 2 on default port 40619 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-11T20:12:40,694 WARN  [IPC Server handler 2 on default port 40619 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-11T20:12:40,694 WARN  [IPC Server handler 2 on default port 40619 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-11T20:12:40,698 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38463 is added to blk_1073741851_1034 (size=12506)
2024-12-11T20:12:40,701 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.50 KB at sequenceid=24 (bloomFilter=true), to=hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/.tmp/info/f4335984ea3a439692479308d3f53619
2024-12-11T20:12:40,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/.tmp/info/f4335984ea3a439692479308d3f53619 as hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/info/f4335984ea3a439692479308d3f53619
2024-12-11T20:12:40,713 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/info/f4335984ea3a439692479308d3f53619, entries=7, sequenceid=24, filesize=12.2 K
2024-12-11T20:12:40,714 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.50 KB/10756, heapSize ~11.48 KB/11760, currentSize=0 B/0 for 5109ba8daed9e21fd7c045352880c060 in 43ms, sequenceid=24, compaction requested=false
2024-12-11T20:12:40,714 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5109ba8daed9e21fd7c045352880c060:

2024-12-11T20:12:40,714 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=22.3 K, sizeToCheck=16.0 K
2024-12-11T20:12:40,715 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-11T20:12:40,715 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/info/f4335984ea3a439692479308d3f53619 because midkey is the same as first or last row
2024-12-11T20:12:41,253 INFO  [regionserver/9fe0640122ec:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:42,033 WARN  [regionserver/9fe0640122ec:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38463,DS-b0a6421d-98c9-48af-ab16-a87f2dea8cbb,DISK]]
2024-12-11T20:12:42,033 INFO  [regionserver/9fe0640122ec:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:42,033 DEBUG [regionserver/9fe0640122ec:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 9fe0640122ec%2C42593%2C1733947942024:(num 1733947958017) roll requested
2024-12-11T20:12:42,034 INFO  [regionserver/9fe0640122ec:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9fe0640122ec%2C42593%2C1733947942024.1733947962034
2024-12-11T20:12:42,037 WARN  [Thread-936 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741852_1035
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:42,038 WARN  [Thread-936 {}] hdfs.DataStreamer(1731): Error Recovery for BP-773969328-172.17.0.2-1733947940734:blk_1073741852_1035 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35127,DS-20198564-3dc6-402d-9d4b-7023e3d1e57b,DISK], DatanodeInfoWithStorage[127.0.0.1:38463,DS-b0a6421d-98c9-48af-ab16-a87f2dea8cbb,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35127,DS-20198564-3dc6-402d-9d4b-7023e3d1e57b,DISK]) is bad.
2024-12-11T20:12:42,038 WARN  [Thread-936 {}] hdfs.DataStreamer(1850): Abandoning BP-773969328-172.17.0.2-1733947940734:blk_1073741852_1035
2024-12-11T20:12:42,038 WARN  [Thread-936 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35127,DS-20198564-3dc6-402d-9d4b-7023e3d1e57b,DISK]
2024-12-11T20:12:42,040 WARN  [Thread-936 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741853_1036
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:46049
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:42,040 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1278250962_22 at /127.0.0.1:40362 [Receiving block BP-773969328-172.17.0.2-1733947940734:blk_1073741853_1036] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/cluster_f21598ea-49df-c0f8-b759-6a4f22597d28/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/cluster_f21598ea-49df-c0f8-b759-6a4f22597d28/data/data10]'}, localName='127.0.0.1:38463', datanodeUuid='ce0d95cd-8c3d-4ad7-8b80-989a1cd7ec68', xmitsInProgress=0}:Exception transferring block BP-773969328-172.17.0.2-1733947940734:blk_1073741853_1036 to mirror 127.0.0.1:46049
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:12:42,041 WARN  [Thread-936 {}] hdfs.DataStreamer(1731): Error Recovery for BP-773969328-172.17.0.2-1733947940734:blk_1073741853_1036 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38463,DS-b0a6421d-98c9-48af-ab16-a87f2dea8cbb,DISK], DatanodeInfoWithStorage[127.0.0.1:46049,DS-cb434015-7991-4b0a-b64b-1a5c770c280f,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:46049,DS-cb434015-7991-4b0a-b64b-1a5c770c280f,DISK]) is bad.
2024-12-11T20:12:42,041 WARN  [Thread-936 {}] hdfs.DataStreamer(1850): Abandoning BP-773969328-172.17.0.2-1733947940734:blk_1073741853_1036
2024-12-11T20:12:42,041 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1278250962_22 at /127.0.0.1:40362 [Receiving block BP-773969328-172.17.0.2-1733947940734:blk_1073741853_1036] {}] datanode.BlockReceiver(316): Block 1073741853 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-12-11T20:12:42,041 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1278250962_22 at /127.0.0.1:40362 [Receiving block BP-773969328-172.17.0.2-1733947940734:blk_1073741853_1036] {}] datanode.DataXceiver(331): 127.0.0.1:38463:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:40362 dst: /127.0.0.1:38463
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:12:42,041 WARN  [Thread-936 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46049,DS-cb434015-7991-4b0a-b64b-1a5c770c280f,DISK]
2024-12-11T20:12:42,042 WARN  [Thread-936 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741854_1037
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:42,042 WARN  [Thread-936 {}] hdfs.DataStreamer(1731): Error Recovery for BP-773969328-172.17.0.2-1733947940734:blk_1073741854_1037 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36207,DS-a9e47326-b9f8-4a53-b473-bfa84b160cb2,DISK], DatanodeInfoWithStorage[127.0.0.1:38463,DS-b0a6421d-98c9-48af-ab16-a87f2dea8cbb,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36207,DS-a9e47326-b9f8-4a53-b473-bfa84b160cb2,DISK]) is bad.
2024-12-11T20:12:42,043 WARN  [Thread-936 {}] hdfs.DataStreamer(1850): Abandoning BP-773969328-172.17.0.2-1733947940734:blk_1073741854_1037
2024-12-11T20:12:42,043 WARN  [Thread-936 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36207,DS-a9e47326-b9f8-4a53-b473-bfa84b160cb2,DISK]
2024-12-11T20:12:42,044 WARN  [Thread-936 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741855_1038
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:42,044 WARN  [Thread-936 {}] hdfs.DataStreamer(1731): Error Recovery for BP-773969328-172.17.0.2-1733947940734:blk_1073741855_1038 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK], DatanodeInfoWithStorage[127.0.0.1:38463,DS-b0a6421d-98c9-48af-ab16-a87f2dea8cbb,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]) is bad.
2024-12-11T20:12:42,044 WARN  [Thread-936 {}] hdfs.DataStreamer(1850): Abandoning BP-773969328-172.17.0.2-1733947940734:blk_1073741855_1038
2024-12-11T20:12:42,045 WARN  [Thread-936 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]
2024-12-11T20:12:42,045 WARN  [IPC Server handler 2 on default port 40619 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-11T20:12:42,045 WARN  [IPC Server handler 2 on default port 40619 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-11T20:12:42,045 WARN  [IPC Server handler 2 on default port 40619 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-11T20:12:42,048 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:12:42,048 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:12:42,048 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:12:42,048 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:12:42,048 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:12:42,049 INFO  [regionserver/9fe0640122ec:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.1733947958017 with entries=25, filesize=25.38 KB; new WAL /user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.1733947962034
2024-12-11T20:12:42,052 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38463 is added to blk_1073741841_1024 (size=25992)
2024-12-11T20:12:42,053 DEBUG [regionserver/9fe0640122ec:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41537:41537)]
2024-12-11T20:12:42,053 DEBUG [regionserver/9fe0640122ec:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.1733947942541 is not closed yet, will try archiving it next time
2024-12-11T20:12:42,053 DEBUG [regionserver/9fe0640122ec:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.1733947958017 is not closed yet, will try archiving it next time
2024-12-11T20:12:42,053 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.1733947953999 to hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/oldWALs/9fe0640122ec%2C42593%2C1733947942024.1733947953999
2024-12-11T20:12:42,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42593 {}] regionserver.HRegion(8855): Flush requested on 5109ba8daed9e21fd7c045352880c060
2024-12-11T20:12:42,094 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5109ba8daed9e21fd7c045352880c060 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-12-11T20:12:42,100 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/.tmp/info/702cafe2606e4ce29c0bc91325d7d2b6 is 1079, key is tmprow/info:/1733947962093/Put/seqid=0
2024-12-11T20:12:42,102 WARN  [Thread-941 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741857_1040
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:42,102 WARN  [Thread-941 {}] hdfs.DataStreamer(1731): Error Recovery for BP-773969328-172.17.0.2-1733947940734:blk_1073741857_1040 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36207,DS-a9e47326-b9f8-4a53-b473-bfa84b160cb2,DISK], DatanodeInfoWithStorage[127.0.0.1:38463,DS-b0a6421d-98c9-48af-ab16-a87f2dea8cbb,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36207,DS-a9e47326-b9f8-4a53-b473-bfa84b160cb2,DISK]) is bad.
2024-12-11T20:12:42,102 WARN  [Thread-941 {}] hdfs.DataStreamer(1850): Abandoning BP-773969328-172.17.0.2-1733947940734:blk_1073741857_1040
2024-12-11T20:12:42,103 WARN  [Thread-941 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36207,DS-a9e47326-b9f8-4a53-b473-bfa84b160cb2,DISK]
2024-12-11T20:12:42,105 WARN  [Thread-941 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741858_1041
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:35127
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:42,105 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1278250962_22 at /127.0.0.1:40380 [Receiving block BP-773969328-172.17.0.2-1733947940734:blk_1073741858_1041] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/cluster_f21598ea-49df-c0f8-b759-6a4f22597d28/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/cluster_f21598ea-49df-c0f8-b759-6a4f22597d28/data/data10]'}, localName='127.0.0.1:38463', datanodeUuid='ce0d95cd-8c3d-4ad7-8b80-989a1cd7ec68', xmitsInProgress=0}:Exception transferring block BP-773969328-172.17.0.2-1733947940734:blk_1073741858_1041 to mirror 127.0.0.1:35127
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:12:42,106 WARN  [Thread-941 {}] hdfs.DataStreamer(1731): Error Recovery for BP-773969328-172.17.0.2-1733947940734:blk_1073741858_1041 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38463,DS-b0a6421d-98c9-48af-ab16-a87f2dea8cbb,DISK], DatanodeInfoWithStorage[127.0.0.1:35127,DS-20198564-3dc6-402d-9d4b-7023e3d1e57b,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:35127,DS-20198564-3dc6-402d-9d4b-7023e3d1e57b,DISK]) is bad.
2024-12-11T20:12:42,106 WARN  [Thread-941 {}] hdfs.DataStreamer(1850): Abandoning BP-773969328-172.17.0.2-1733947940734:blk_1073741858_1041
2024-12-11T20:12:42,106 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1278250962_22 at /127.0.0.1:40380 [Receiving block BP-773969328-172.17.0.2-1733947940734:blk_1073741858_1041] {}] datanode.BlockReceiver(316): Block 1073741858 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-11T20:12:42,106 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1278250962_22 at /127.0.0.1:40380 [Receiving block BP-773969328-172.17.0.2-1733947940734:blk_1073741858_1041] {}] datanode.DataXceiver(331): 127.0.0.1:38463:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:40380 dst: /127.0.0.1:38463
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:12:42,106 WARN  [Thread-941 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35127,DS-20198564-3dc6-402d-9d4b-7023e3d1e57b,DISK]
2024-12-11T20:12:42,107 WARN  [Thread-941 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741859_1042
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:42,107 WARN  [Thread-941 {}] hdfs.DataStreamer(1731): Error Recovery for BP-773969328-172.17.0.2-1733947940734:blk_1073741859_1042 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46049,DS-cb434015-7991-4b0a-b64b-1a5c770c280f,DISK], DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:46049,DS-cb434015-7991-4b0a-b64b-1a5c770c280f,DISK]) is bad.
2024-12-11T20:12:42,108 WARN  [Thread-941 {}] hdfs.DataStreamer(1850): Abandoning BP-773969328-172.17.0.2-1733947940734:blk_1073741859_1042
2024-12-11T20:12:42,108 WARN  [Thread-941 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46049,DS-cb434015-7991-4b0a-b64b-1a5c770c280f,DISK]
2024-12-11T20:12:42,109 WARN  [Thread-941 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741860_1043
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:42,109 WARN  [Thread-941 {}] hdfs.DataStreamer(1731): Error Recovery for BP-773969328-172.17.0.2-1733947940734:blk_1073741860_1043 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK], DatanodeInfoWithStorage[127.0.0.1:38463,DS-b0a6421d-98c9-48af-ab16-a87f2dea8cbb,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]) is bad.
2024-12-11T20:12:42,109 WARN  [Thread-941 {}] hdfs.DataStreamer(1850): Abandoning BP-773969328-172.17.0.2-1733947940734:blk_1073741860_1043
2024-12-11T20:12:42,110 WARN  [Thread-941 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]
2024-12-11T20:12:42,111 WARN  [IPC Server handler 4 on default port 40619 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-11T20:12:42,111 WARN  [IPC Server handler 4 on default port 40619 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-11T20:12:42,111 WARN  [IPC Server handler 4 on default port 40619 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-11T20:12:42,114 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38463 is added to blk_1073741861_1044 (size=6027)
2024-12-11T20:12:42,118 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:42,453 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.1733947942541 is not closed yet, will try archiving it next time
2024-12-11T20:12:42,515 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/.tmp/info/702cafe2606e4ce29c0bc91325d7d2b6
2024-12-11T20:12:42,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/.tmp/info/702cafe2606e4ce29c0bc91325d7d2b6 as hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/info/702cafe2606e4ce29c0bc91325d7d2b6
2024-12-11T20:12:42,528 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/info/702cafe2606e4ce29c0bc91325d7d2b6, entries=1, sequenceid=34, filesize=5.9 K
2024-12-11T20:12:42,530 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for 5109ba8daed9e21fd7c045352880c060 in 436ms, sequenceid=34, compaction requested=true
2024-12-11T20:12:42,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5109ba8daed9e21fd7c045352880c060:

2024-12-11T20:12:42,530 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.2 K, sizeToCheck=16.0 K
2024-12-11T20:12:42,530 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-11T20:12:42,530 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/info/f4335984ea3a439692479308d3f53619 because midkey is the same as first or last row
2024-12-11T20:12:42,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5109ba8daed9e21fd7c045352880c060:info, priority=-2147483648, current under compaction store size is 1
2024-12-11T20:12:42,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-11T20:12:42,531 DEBUG [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-11T20:12:42,532 DEBUG [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 28880 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-11T20:12:42,532 DEBUG [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] regionserver.HStore(1541): 5109ba8daed9e21fd7c045352880c060/info is initiating minor compaction (all files)
2024-12-11T20:12:42,532 INFO  [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5109ba8daed9e21fd7c045352880c060/info in TestLogRolling-testLogRollOnDatanodeDeath,,1733947943302.5109ba8daed9e21fd7c045352880c060.
2024-12-11T20:12:42,532 INFO  [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/info/c715426bd5aa4aa0af6e97e1bd20c1f1, hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/info/f4335984ea3a439692479308d3f53619, hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/info/702cafe2606e4ce29c0bc91325d7d2b6] into tmpdir=hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/.tmp, totalSize=28.2 K
2024-12-11T20:12:42,533 DEBUG [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] compactions.Compactor(225): Compacting c715426bd5aa4aa0af6e97e1bd20c1f1, keycount=5, bloomtype=ROW, size=10.1 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733947956025
2024-12-11T20:12:42,533 DEBUG [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4335984ea3a439692479308d3f53619, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=24, earliestPutTs=1733947960050
2024-12-11T20:12:42,534 DEBUG [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 702cafe2606e4ce29c0bc91325d7d2b6, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1733947962093
2024-12-11T20:12:42,548 INFO  [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5109ba8daed9e21fd7c045352880c060#info#compaction#21 average throughput is 4.10 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-11T20:12:42,548 DEBUG [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/.tmp/info/959cda3dfb4b406f8b936903be745d84 is 1080, key is row0002/info:/1733947956025/Put/seqid=0
2024-12-11T20:12:42,550 WARN  [Thread-948 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741862_1045
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:42,550 WARN  [Thread-948 {}] hdfs.DataStreamer(1731): Error Recovery for BP-773969328-172.17.0.2-1733947940734:blk_1073741862_1045 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36207,DS-a9e47326-b9f8-4a53-b473-bfa84b160cb2,DISK], DatanodeInfoWithStorage[127.0.0.1:35127,DS-20198564-3dc6-402d-9d4b-7023e3d1e57b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36207,DS-a9e47326-b9f8-4a53-b473-bfa84b160cb2,DISK]) is bad.
2024-12-11T20:12:42,550 WARN  [Thread-948 {}] hdfs.DataStreamer(1850): Abandoning BP-773969328-172.17.0.2-1733947940734:blk_1073741862_1045
2024-12-11T20:12:42,551 WARN  [Thread-948 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36207,DS-a9e47326-b9f8-4a53-b473-bfa84b160cb2,DISK]
2024-12-11T20:12:42,552 WARN  [Thread-948 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741863_1046
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:42,552 WARN  [Thread-948 {}] hdfs.DataStreamer(1731): Error Recovery for BP-773969328-172.17.0.2-1733947940734:blk_1073741863_1046 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35127,DS-20198564-3dc6-402d-9d4b-7023e3d1e57b,DISK], DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35127,DS-20198564-3dc6-402d-9d4b-7023e3d1e57b,DISK]) is bad.
2024-12-11T20:12:42,552 WARN  [Thread-948 {}] hdfs.DataStreamer(1850): Abandoning BP-773969328-172.17.0.2-1733947940734:blk_1073741863_1046
2024-12-11T20:12:42,552 WARN  [Thread-948 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35127,DS-20198564-3dc6-402d-9d4b-7023e3d1e57b,DISK]
2024-12-11T20:12:42,554 WARN  [Thread-948 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741864_1047
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:34893
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:42,554 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1278250962_22 at /127.0.0.1:40432 [Receiving block BP-773969328-172.17.0.2-1733947940734:blk_1073741864_1047] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/cluster_f21598ea-49df-c0f8-b759-6a4f22597d28/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/cluster_f21598ea-49df-c0f8-b759-6a4f22597d28/data/data10]'}, localName='127.0.0.1:38463', datanodeUuid='ce0d95cd-8c3d-4ad7-8b80-989a1cd7ec68', xmitsInProgress=0}:Exception transferring block BP-773969328-172.17.0.2-1733947940734:blk_1073741864_1047 to mirror 127.0.0.1:34893
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:12:42,555 WARN  [Thread-948 {}] hdfs.DataStreamer(1731): Error Recovery for BP-773969328-172.17.0.2-1733947940734:blk_1073741864_1047 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38463,DS-b0a6421d-98c9-48af-ab16-a87f2dea8cbb,DISK], DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]) is bad.
2024-12-11T20:12:42,555 WARN  [Thread-948 {}] hdfs.DataStreamer(1850): Abandoning BP-773969328-172.17.0.2-1733947940734:blk_1073741864_1047
2024-12-11T20:12:42,555 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1278250962_22 at /127.0.0.1:40432 [Receiving block BP-773969328-172.17.0.2-1733947940734:blk_1073741864_1047] {}] datanode.BlockReceiver(316): Block 1073741864 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-11T20:12:42,555 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1278250962_22 at /127.0.0.1:40432 [Receiving block BP-773969328-172.17.0.2-1733947940734:blk_1073741864_1047] {}] datanode.DataXceiver(331): 127.0.0.1:38463:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:40432 dst: /127.0.0.1:38463
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:12:42,555 WARN  [Thread-948 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]
2024-12-11T20:12:42,557 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1278250962_22 at /127.0.0.1:40436 [Receiving block BP-773969328-172.17.0.2-1733947940734:blk_1073741865_1048] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/cluster_f21598ea-49df-c0f8-b759-6a4f22597d28/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/cluster_f21598ea-49df-c0f8-b759-6a4f22597d28/data/data10]'}, localName='127.0.0.1:38463', datanodeUuid='ce0d95cd-8c3d-4ad7-8b80-989a1cd7ec68', xmitsInProgress=0}:Exception transferring block BP-773969328-172.17.0.2-1733947940734:blk_1073741865_1048 to mirror 127.0.0.1:46049
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:12:42,557 WARN  [Thread-948 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741865_1048
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:46049
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:42,557 WARN  [Thread-948 {}] hdfs.DataStreamer(1731): Error Recovery for BP-773969328-172.17.0.2-1733947940734:blk_1073741865_1048 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38463,DS-b0a6421d-98c9-48af-ab16-a87f2dea8cbb,DISK], DatanodeInfoWithStorage[127.0.0.1:46049,DS-cb434015-7991-4b0a-b64b-1a5c770c280f,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:46049,DS-cb434015-7991-4b0a-b64b-1a5c770c280f,DISK]) is bad.
2024-12-11T20:12:42,557 WARN  [Thread-948 {}] hdfs.DataStreamer(1850): Abandoning BP-773969328-172.17.0.2-1733947940734:blk_1073741865_1048
2024-12-11T20:12:42,557 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1278250962_22 at /127.0.0.1:40436 [Receiving block BP-773969328-172.17.0.2-1733947940734:blk_1073741865_1048] {}] datanode.BlockReceiver(316): Block 1073741865 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-11T20:12:42,558 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1278250962_22 at /127.0.0.1:40436 [Receiving block BP-773969328-172.17.0.2-1733947940734:blk_1073741865_1048] {}] datanode.DataXceiver(331): 127.0.0.1:38463:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:40436 dst: /127.0.0.1:38463
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:12:42,558 WARN  [Thread-948 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46049,DS-cb434015-7991-4b0a-b64b-1a5c770c280f,DISK]
2024-12-11T20:12:42,558 WARN  [IPC Server handler 0 on default port 40619 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-11T20:12:42,559 WARN  [IPC Server handler 0 on default port 40619 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-11T20:12:42,559 WARN  [IPC Server handler 0 on default port 40619 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-11T20:12:42,562 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38463 is added to blk_1073741866_1049 (size=17994)
2024-12-11T20:12:42,970 DEBUG [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/.tmp/info/959cda3dfb4b406f8b936903be745d84 as hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/info/959cda3dfb4b406f8b936903be745d84
2024-12-11T20:12:42,971 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@74770451[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:38463, datanodeUuid=ce0d95cd-8c3d-4ad7-8b80-989a1cd7ec68, infoPort=41537, infoSecurePort=0, ipcPort=42607, storageInfo=lv=-57;cid=testClusterID;nsid=1349632812;c=1733947940734):Failed to transfer BP-773969328-172.17.0.2-1733947940734:blk_1073741846_1029 to 127.0.0.1:35127 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:12:42,971 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@7ab0f4af[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:38463, datanodeUuid=ce0d95cd-8c3d-4ad7-8b80-989a1cd7ec68, infoPort=41537, infoSecurePort=0, ipcPort=42607, storageInfo=lv=-57;cid=testClusterID;nsid=1349632812;c=1733947940734):Failed to transfer BP-773969328-172.17.0.2-1733947940734:blk_1073741851_1034 to 127.0.0.1:34893 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:12:42,978 INFO  [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5109ba8daed9e21fd7c045352880c060/info of 5109ba8daed9e21fd7c045352880c060 into 959cda3dfb4b406f8b936903be745d84(size=17.6 K), total size for store is 17.6 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-11T20:12:42,978 DEBUG [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5109ba8daed9e21fd7c045352880c060:

2024-12-11T20:12:42,978 INFO  [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1733947943302.5109ba8daed9e21fd7c045352880c060., storeName=5109ba8daed9e21fd7c045352880c060/info, priority=13, startTime=1733947962530; duration=0sec
2024-12-11T20:12:42,978 DEBUG [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K
2024-12-11T20:12:42,979 DEBUG [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-11T20:12:42,979 DEBUG [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/info/959cda3dfb4b406f8b936903be745d84 because midkey is the same as first or last row
2024-12-11T20:12:42,979 DEBUG [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K
2024-12-11T20:12:42,979 DEBUG [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-11T20:12:42,979 DEBUG [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/info/959cda3dfb4b406f8b936903be745d84 because midkey is the same as first or last row
2024-12-11T20:12:42,979 DEBUG [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K
2024-12-11T20:12:42,979 DEBUG [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-11T20:12:42,979 DEBUG [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/info/959cda3dfb4b406f8b936903be745d84 because midkey is the same as first or last row
2024-12-11T20:12:42,979 DEBUG [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-11T20:12:42,979 DEBUG [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5109ba8daed9e21fd7c045352880c060:info
2024-12-11T20:12:43,253 INFO  [regionserver/9fe0640122ec:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:43,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42593 {}] regionserver.HRegion(8855): Flush requested on 5109ba8daed9e21fd7c045352880c060
2024-12-11T20:12:43,514 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5109ba8daed9e21fd7c045352880c060 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-12-11T20:12:43,519 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/.tmp/info/3577146a7f4e49b787498369a919c703 is 1079, key is tmprow/info:/1733947963512/Put/seqid=0
2024-12-11T20:12:43,521 WARN  [Thread-958 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741867_1050
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:43,521 WARN  [Thread-958 {}] hdfs.DataStreamer(1731): Error Recovery for BP-773969328-172.17.0.2-1733947940734:blk_1073741867_1050 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK], DatanodeInfoWithStorage[127.0.0.1:36207,DS-a9e47326-b9f8-4a53-b473-bfa84b160cb2,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]) is bad.
2024-12-11T20:12:43,521 WARN  [Thread-958 {}] hdfs.DataStreamer(1850): Abandoning BP-773969328-172.17.0.2-1733947940734:blk_1073741867_1050
2024-12-11T20:12:43,522 WARN  [Thread-958 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]
2024-12-11T20:12:43,523 WARN  [Thread-958 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741868_1051
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:43,524 WARN  [Thread-958 {}] hdfs.DataStreamer(1731): Error Recovery for BP-773969328-172.17.0.2-1733947940734:blk_1073741868_1051 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36207,DS-a9e47326-b9f8-4a53-b473-bfa84b160cb2,DISK], DatanodeInfoWithStorage[127.0.0.1:38463,DS-b0a6421d-98c9-48af-ab16-a87f2dea8cbb,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36207,DS-a9e47326-b9f8-4a53-b473-bfa84b160cb2,DISK]) is bad.
2024-12-11T20:12:43,524 WARN  [Thread-958 {}] hdfs.DataStreamer(1850): Abandoning BP-773969328-172.17.0.2-1733947940734:blk_1073741868_1051
2024-12-11T20:12:43,524 WARN  [Thread-958 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36207,DS-a9e47326-b9f8-4a53-b473-bfa84b160cb2,DISK]
2024-12-11T20:12:43,526 WARN  [Thread-958 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741869_1052
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:35127
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:43,526 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1278250962_22 at /127.0.0.1:40460 [Receiving block BP-773969328-172.17.0.2-1733947940734:blk_1073741869_1052] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/cluster_f21598ea-49df-c0f8-b759-6a4f22597d28/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/cluster_f21598ea-49df-c0f8-b759-6a4f22597d28/data/data10]'}, localName='127.0.0.1:38463', datanodeUuid='ce0d95cd-8c3d-4ad7-8b80-989a1cd7ec68', xmitsInProgress=0}:Exception transferring block BP-773969328-172.17.0.2-1733947940734:blk_1073741869_1052 to mirror 127.0.0.1:35127
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:12:43,526 WARN  [Thread-958 {}] hdfs.DataStreamer(1731): Error Recovery for BP-773969328-172.17.0.2-1733947940734:blk_1073741869_1052 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38463,DS-b0a6421d-98c9-48af-ab16-a87f2dea8cbb,DISK], DatanodeInfoWithStorage[127.0.0.1:35127,DS-20198564-3dc6-402d-9d4b-7023e3d1e57b,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:35127,DS-20198564-3dc6-402d-9d4b-7023e3d1e57b,DISK]) is bad.
2024-12-11T20:12:43,526 WARN  [Thread-958 {}] hdfs.DataStreamer(1850): Abandoning BP-773969328-172.17.0.2-1733947940734:blk_1073741869_1052
2024-12-11T20:12:43,526 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1278250962_22 at /127.0.0.1:40460 [Receiving block BP-773969328-172.17.0.2-1733947940734:blk_1073741869_1052] {}] datanode.BlockReceiver(316): Block 1073741869 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-11T20:12:43,526 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1278250962_22 at /127.0.0.1:40460 [Receiving block BP-773969328-172.17.0.2-1733947940734:blk_1073741869_1052] {}] datanode.DataXceiver(331): 127.0.0.1:38463:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:40460 dst: /127.0.0.1:38463
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:12:43,527 WARN  [Thread-958 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35127,DS-20198564-3dc6-402d-9d4b-7023e3d1e57b,DISK]
2024-12-11T20:12:43,529 WARN  [Thread-958 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741870_1053
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:46049
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:43,529 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1278250962_22 at /127.0.0.1:40474 [Receiving block BP-773969328-172.17.0.2-1733947940734:blk_1073741870_1053] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/cluster_f21598ea-49df-c0f8-b759-6a4f22597d28/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/cluster_f21598ea-49df-c0f8-b759-6a4f22597d28/data/data10]'}, localName='127.0.0.1:38463', datanodeUuid='ce0d95cd-8c3d-4ad7-8b80-989a1cd7ec68', xmitsInProgress=0}:Exception transferring block BP-773969328-172.17.0.2-1733947940734:blk_1073741870_1053 to mirror 127.0.0.1:46049
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:12:43,529 WARN  [Thread-958 {}] hdfs.DataStreamer(1731): Error Recovery for BP-773969328-172.17.0.2-1733947940734:blk_1073741870_1053 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38463,DS-b0a6421d-98c9-48af-ab16-a87f2dea8cbb,DISK], DatanodeInfoWithStorage[127.0.0.1:46049,DS-cb434015-7991-4b0a-b64b-1a5c770c280f,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:46049,DS-cb434015-7991-4b0a-b64b-1a5c770c280f,DISK]) is bad.
2024-12-11T20:12:43,529 WARN  [Thread-958 {}] hdfs.DataStreamer(1850): Abandoning BP-773969328-172.17.0.2-1733947940734:blk_1073741870_1053
2024-12-11T20:12:43,529 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1278250962_22 at /127.0.0.1:40474 [Receiving block BP-773969328-172.17.0.2-1733947940734:blk_1073741870_1053] {}] datanode.BlockReceiver(316): Block 1073741870 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-11T20:12:43,529 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1278250962_22 at /127.0.0.1:40474 [Receiving block BP-773969328-172.17.0.2-1733947940734:blk_1073741870_1053] {}] datanode.DataXceiver(331): 127.0.0.1:38463:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:40474 dst: /127.0.0.1:38463
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:12:43,530 WARN  [Thread-958 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46049,DS-cb434015-7991-4b0a-b64b-1a5c770c280f,DISK]
2024-12-11T20:12:43,530 WARN  [IPC Server handler 0 on default port 40619 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-11T20:12:43,530 WARN  [IPC Server handler 0 on default port 40619 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-11T20:12:43,530 WARN  [IPC Server handler 0 on default port 40619 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-11T20:12:43,535 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38463 is added to blk_1073741871_1054 (size=6027)
2024-12-11T20:12:43,936 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/.tmp/info/3577146a7f4e49b787498369a919c703
2024-12-11T20:12:43,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/.tmp/info/3577146a7f4e49b787498369a919c703 as hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/info/3577146a7f4e49b787498369a919c703
2024-12-11T20:12:43,949 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/info/3577146a7f4e49b787498369a919c703, entries=1, sequenceid=45, filesize=5.9 K
2024-12-11T20:12:43,950 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for 5109ba8daed9e21fd7c045352880c060 in 436ms, sequenceid=45, compaction requested=false
2024-12-11T20:12:43,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5109ba8daed9e21fd7c045352880c060:

2024-12-11T20:12:43,950 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=23.5 K, sizeToCheck=16.0 K
2024-12-11T20:12:43,950 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-11T20:12:43,951 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/info/959cda3dfb4b406f8b936903be745d84 because midkey is the same as first or last row
2024-12-11T20:12:44,053 WARN  [regionserver/9fe0640122ec:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38463,DS-b0a6421d-98c9-48af-ab16-a87f2dea8cbb,DISK]]
2024-12-11T20:12:44,054 INFO  [regionserver/9fe0640122ec:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:44,054 DEBUG [regionserver/9fe0640122ec:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 9fe0640122ec%2C42593%2C1733947942024:(num 1733947962034) roll requested
2024-12-11T20:12:44,054 INFO  [regionserver/9fe0640122ec:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9fe0640122ec%2C42593%2C1733947942024.1733947964054
2024-12-11T20:12:44,058 WARN  [Thread-964 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741872_1055
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:36207
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:44,058 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1278250962_22 at /127.0.0.1:40492 [Receiving block BP-773969328-172.17.0.2-1733947940734:blk_1073741872_1055] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/cluster_f21598ea-49df-c0f8-b759-6a4f22597d28/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/cluster_f21598ea-49df-c0f8-b759-6a4f22597d28/data/data10]'}, localName='127.0.0.1:38463', datanodeUuid='ce0d95cd-8c3d-4ad7-8b80-989a1cd7ec68', xmitsInProgress=0}:Exception transferring block BP-773969328-172.17.0.2-1733947940734:blk_1073741872_1055 to mirror 127.0.0.1:36207
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:12:44,058 WARN  [Thread-964 {}] hdfs.DataStreamer(1731): Error Recovery for BP-773969328-172.17.0.2-1733947940734:blk_1073741872_1055 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38463,DS-b0a6421d-98c9-48af-ab16-a87f2dea8cbb,DISK], DatanodeInfoWithStorage[127.0.0.1:36207,DS-a9e47326-b9f8-4a53-b473-bfa84b160cb2,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:36207,DS-a9e47326-b9f8-4a53-b473-bfa84b160cb2,DISK]) is bad.
2024-12-11T20:12:44,058 WARN  [Thread-964 {}] hdfs.DataStreamer(1850): Abandoning BP-773969328-172.17.0.2-1733947940734:blk_1073741872_1055
2024-12-11T20:12:44,058 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1278250962_22 at /127.0.0.1:40492 [Receiving block BP-773969328-172.17.0.2-1733947940734:blk_1073741872_1055] {}] datanode.BlockReceiver(316): Block 1073741872 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-12-11T20:12:44,059 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1278250962_22 at /127.0.0.1:40492 [Receiving block BP-773969328-172.17.0.2-1733947940734:blk_1073741872_1055] {}] datanode.DataXceiver(331): 127.0.0.1:38463:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:40492 dst: /127.0.0.1:38463
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:12:44,059 WARN  [Thread-964 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36207,DS-a9e47326-b9f8-4a53-b473-bfa84b160cb2,DISK]
2024-12-11T20:12:44,060 WARN  [Thread-964 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741873_1056
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:44,060 WARN  [Thread-964 {}] hdfs.DataStreamer(1731): Error Recovery for BP-773969328-172.17.0.2-1733947940734:blk_1073741873_1056 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46049,DS-cb434015-7991-4b0a-b64b-1a5c770c280f,DISK], DatanodeInfoWithStorage[127.0.0.1:38463,DS-b0a6421d-98c9-48af-ab16-a87f2dea8cbb,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:46049,DS-cb434015-7991-4b0a-b64b-1a5c770c280f,DISK]) is bad.
2024-12-11T20:12:44,060 WARN  [Thread-964 {}] hdfs.DataStreamer(1850): Abandoning BP-773969328-172.17.0.2-1733947940734:blk_1073741873_1056
2024-12-11T20:12:44,060 WARN  [Thread-964 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46049,DS-cb434015-7991-4b0a-b64b-1a5c770c280f,DISK]
2024-12-11T20:12:44,061 WARN  [Thread-964 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741874_1057
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:44,062 WARN  [Thread-964 {}] hdfs.DataStreamer(1731): Error Recovery for BP-773969328-172.17.0.2-1733947940734:blk_1073741874_1057 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35127,DS-20198564-3dc6-402d-9d4b-7023e3d1e57b,DISK], DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35127,DS-20198564-3dc6-402d-9d4b-7023e3d1e57b,DISK]) is bad.
2024-12-11T20:12:44,062 WARN  [Thread-964 {}] hdfs.DataStreamer(1850): Abandoning BP-773969328-172.17.0.2-1733947940734:blk_1073741874_1057
2024-12-11T20:12:44,062 WARN  [Thread-964 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35127,DS-20198564-3dc6-402d-9d4b-7023e3d1e57b,DISK]
2024-12-11T20:12:44,064 WARN  [Thread-964 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741875_1058
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:34893
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:44,064 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1278250962_22 at /127.0.0.1:40500 [Receiving block BP-773969328-172.17.0.2-1733947940734:blk_1073741875_1058] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/cluster_f21598ea-49df-c0f8-b759-6a4f22597d28/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/cluster_f21598ea-49df-c0f8-b759-6a4f22597d28/data/data10]'}, localName='127.0.0.1:38463', datanodeUuid='ce0d95cd-8c3d-4ad7-8b80-989a1cd7ec68', xmitsInProgress=0}:Exception transferring block BP-773969328-172.17.0.2-1733947940734:blk_1073741875_1058 to mirror 127.0.0.1:34893
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:12:44,064 WARN  [Thread-964 {}] hdfs.DataStreamer(1731): Error Recovery for BP-773969328-172.17.0.2-1733947940734:blk_1073741875_1058 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38463,DS-b0a6421d-98c9-48af-ab16-a87f2dea8cbb,DISK], DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]) is bad.
2024-12-11T20:12:44,064 WARN  [Thread-964 {}] hdfs.DataStreamer(1850): Abandoning BP-773969328-172.17.0.2-1733947940734:blk_1073741875_1058
2024-12-11T20:12:44,065 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1278250962_22 at /127.0.0.1:40500 [Receiving block BP-773969328-172.17.0.2-1733947940734:blk_1073741875_1058] {}] datanode.BlockReceiver(316): Block 1073741875 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-12-11T20:12:44,065 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1278250962_22 at /127.0.0.1:40500 [Receiving block BP-773969328-172.17.0.2-1733947940734:blk_1073741875_1058] {}] datanode.DataXceiver(331): 127.0.0.1:38463:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:40500 dst: /127.0.0.1:38463
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:12:44,065 WARN  [Thread-964 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]
2024-12-11T20:12:44,065 WARN  [IPC Server handler 0 on default port 40619 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-11T20:12:44,066 WARN  [IPC Server handler 0 on default port 40619 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-11T20:12:44,066 WARN  [IPC Server handler 0 on default port 40619 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-11T20:12:44,068 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:12:44,068 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:12:44,068 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:12:44,068 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:12:44,068 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:12:44,068 INFO  [regionserver/9fe0640122ec:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.1733947962034 with entries=15, filesize=13.26 KB; new WAL /user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.1733947964054
2024-12-11T20:12:44,069 DEBUG [regionserver/9fe0640122ec:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41537:41537)]
2024-12-11T20:12:44,069 DEBUG [regionserver/9fe0640122ec:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.1733947942541 is not closed yet, will try archiving it next time
2024-12-11T20:12:44,069 DEBUG [regionserver/9fe0640122ec:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.1733947962034 is not closed yet, will try archiving it next time
2024-12-11T20:12:44,069 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.1733947958017 to hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/oldWALs/9fe0640122ec%2C42593%2C1733947942024.1733947958017
2024-12-11T20:12:44,070 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38463 is added to blk_1073741856_1039 (size=13591)
2024-12-11T20:12:44,071 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.1733947942541 is not closed yet, will try archiving it next time
2024-12-11T20:12:44,119 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:44,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42593 {}] regionserver.HRegion(8855): Flush requested on 5109ba8daed9e21fd7c045352880c060
2024-12-11T20:12:44,933 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5109ba8daed9e21fd7c045352880c060 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-12-11T20:12:44,938 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/.tmp/info/ac10ce55978746efa451b84291a89ce9 is 1079, key is tmprow/info:/1733947964932/Put/seqid=0
2024-12-11T20:12:44,939 WARN  [Thread-970 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741877_1060
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:44,940 WARN  [Thread-970 {}] hdfs.DataStreamer(1731): Error Recovery for BP-773969328-172.17.0.2-1733947940734:blk_1073741877_1060 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46049,DS-cb434015-7991-4b0a-b64b-1a5c770c280f,DISK], DatanodeInfoWithStorage[127.0.0.1:36207,DS-a9e47326-b9f8-4a53-b473-bfa84b160cb2,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:46049,DS-cb434015-7991-4b0a-b64b-1a5c770c280f,DISK]) is bad.
2024-12-11T20:12:44,940 WARN  [Thread-970 {}] hdfs.DataStreamer(1850): Abandoning BP-773969328-172.17.0.2-1733947940734:blk_1073741877_1060
2024-12-11T20:12:44,940 WARN  [Thread-970 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46049,DS-cb434015-7991-4b0a-b64b-1a5c770c280f,DISK]
2024-12-11T20:12:44,942 WARN  [Thread-970 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741878_1061
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:44,942 WARN  [Thread-970 {}] hdfs.DataStreamer(1731): Error Recovery for BP-773969328-172.17.0.2-1733947940734:blk_1073741878_1061 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK], DatanodeInfoWithStorage[127.0.0.1:38463,DS-b0a6421d-98c9-48af-ab16-a87f2dea8cbb,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]) is bad.
2024-12-11T20:12:44,942 WARN  [Thread-970 {}] hdfs.DataStreamer(1850): Abandoning BP-773969328-172.17.0.2-1733947940734:blk_1073741878_1061
2024-12-11T20:12:44,942 WARN  [Thread-970 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]
2024-12-11T20:12:44,944 WARN  [Thread-970 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741879_1062
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:36207
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:44,944 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1278250962_22 at /127.0.0.1:49724 [Receiving block BP-773969328-172.17.0.2-1733947940734:blk_1073741879_1062] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/cluster_f21598ea-49df-c0f8-b759-6a4f22597d28/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/cluster_f21598ea-49df-c0f8-b759-6a4f22597d28/data/data10]'}, localName='127.0.0.1:38463', datanodeUuid='ce0d95cd-8c3d-4ad7-8b80-989a1cd7ec68', xmitsInProgress=0}:Exception transferring block BP-773969328-172.17.0.2-1733947940734:blk_1073741879_1062 to mirror 127.0.0.1:36207
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:12:44,944 WARN  [Thread-970 {}] hdfs.DataStreamer(1731): Error Recovery for BP-773969328-172.17.0.2-1733947940734:blk_1073741879_1062 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38463,DS-b0a6421d-98c9-48af-ab16-a87f2dea8cbb,DISK], DatanodeInfoWithStorage[127.0.0.1:36207,DS-a9e47326-b9f8-4a53-b473-bfa84b160cb2,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:36207,DS-a9e47326-b9f8-4a53-b473-bfa84b160cb2,DISK]) is bad.
2024-12-11T20:12:44,945 WARN  [Thread-970 {}] hdfs.DataStreamer(1850): Abandoning BP-773969328-172.17.0.2-1733947940734:blk_1073741879_1062
2024-12-11T20:12:44,945 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1278250962_22 at /127.0.0.1:49724 [Receiving block BP-773969328-172.17.0.2-1733947940734:blk_1073741879_1062] {}] datanode.BlockReceiver(316): Block 1073741879 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-11T20:12:44,945 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1278250962_22 at /127.0.0.1:49724 [Receiving block BP-773969328-172.17.0.2-1733947940734:blk_1073741879_1062] {}] datanode.DataXceiver(331): 127.0.0.1:38463:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:49724 dst: /127.0.0.1:38463
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:12:44,945 WARN  [Thread-970 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36207,DS-a9e47326-b9f8-4a53-b473-bfa84b160cb2,DISK]
2024-12-11T20:12:44,946 WARN  [Thread-970 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741880_1063
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:44,946 WARN  [Thread-970 {}] hdfs.DataStreamer(1731): Error Recovery for BP-773969328-172.17.0.2-1733947940734:blk_1073741880_1063 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35127,DS-20198564-3dc6-402d-9d4b-7023e3d1e57b,DISK], DatanodeInfoWithStorage[127.0.0.1:38463,DS-b0a6421d-98c9-48af-ab16-a87f2dea8cbb,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35127,DS-20198564-3dc6-402d-9d4b-7023e3d1e57b,DISK]) is bad.
2024-12-11T20:12:44,946 WARN  [Thread-970 {}] hdfs.DataStreamer(1850): Abandoning BP-773969328-172.17.0.2-1733947940734:blk_1073741880_1063
2024-12-11T20:12:44,947 WARN  [Thread-970 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35127,DS-20198564-3dc6-402d-9d4b-7023e3d1e57b,DISK]
2024-12-11T20:12:44,947 WARN  [IPC Server handler 4 on default port 40619 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-11T20:12:44,947 WARN  [IPC Server handler 4 on default port 40619 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-11T20:12:44,948 WARN  [IPC Server handler 4 on default port 40619 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-11T20:12:44,950 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38463 is added to blk_1073741881_1064 (size=6027)
2024-12-11T20:12:45,253 INFO  [regionserver/9fe0640122ec:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:45,351 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/.tmp/info/ac10ce55978746efa451b84291a89ce9
2024-12-11T20:12:45,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/.tmp/info/ac10ce55978746efa451b84291a89ce9 as hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/info/ac10ce55978746efa451b84291a89ce9
2024-12-11T20:12:45,364 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/info/ac10ce55978746efa451b84291a89ce9, entries=1, sequenceid=55, filesize=5.9 K
2024-12-11T20:12:45,365 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for 5109ba8daed9e21fd7c045352880c060 in 432ms, sequenceid=55, compaction requested=true
2024-12-11T20:12:45,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5109ba8daed9e21fd7c045352880c060:

2024-12-11T20:12:45,365 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=29.3 K, sizeToCheck=16.0 K
2024-12-11T20:12:45,365 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-11T20:12:45,365 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/info/959cda3dfb4b406f8b936903be745d84 because midkey is the same as first or last row
2024-12-11T20:12:45,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5109ba8daed9e21fd7c045352880c060:info, priority=-2147483648, current under compaction store size is 1
2024-12-11T20:12:45,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-11T20:12:45,365 DEBUG [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-11T20:12:45,367 DEBUG [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 30048 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-11T20:12:45,367 DEBUG [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] regionserver.HStore(1541): 5109ba8daed9e21fd7c045352880c060/info is initiating minor compaction (all files)
2024-12-11T20:12:45,367 INFO  [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5109ba8daed9e21fd7c045352880c060/info in TestLogRolling-testLogRollOnDatanodeDeath,,1733947943302.5109ba8daed9e21fd7c045352880c060.
2024-12-11T20:12:45,367 INFO  [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/info/959cda3dfb4b406f8b936903be745d84, hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/info/3577146a7f4e49b787498369a919c703, hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/info/ac10ce55978746efa451b84291a89ce9] into tmpdir=hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/.tmp, totalSize=29.3 K
2024-12-11T20:12:45,367 DEBUG [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 959cda3dfb4b406f8b936903be745d84, keycount=12, bloomtype=ROW, size=17.6 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1733947956025
2024-12-11T20:12:45,368 DEBUG [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3577146a7f4e49b787498369a919c703, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1733947963512
2024-12-11T20:12:45,368 DEBUG [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac10ce55978746efa451b84291a89ce9, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1733947964932
2024-12-11T20:12:45,383 INFO  [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5109ba8daed9e21fd7c045352880c060#info#compaction#24 average throughput is 12.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-11T20:12:45,383 DEBUG [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/.tmp/info/7aaef61b9f1e4df8a6a16ff64526d42f is 1080, key is row0002/info:/1733947956025/Put/seqid=0
2024-12-11T20:12:45,385 WARN  [Thread-975 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741882_1065
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:45,385 WARN  [Thread-975 {}] hdfs.DataStreamer(1731): Error Recovery for BP-773969328-172.17.0.2-1733947940734:blk_1073741882_1065 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36207,DS-a9e47326-b9f8-4a53-b473-bfa84b160cb2,DISK], DatanodeInfoWithStorage[127.0.0.1:38463,DS-b0a6421d-98c9-48af-ab16-a87f2dea8cbb,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36207,DS-a9e47326-b9f8-4a53-b473-bfa84b160cb2,DISK]) is bad.
2024-12-11T20:12:45,385 WARN  [Thread-975 {}] hdfs.DataStreamer(1850): Abandoning BP-773969328-172.17.0.2-1733947940734:blk_1073741882_1065
2024-12-11T20:12:45,386 WARN  [Thread-975 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36207,DS-a9e47326-b9f8-4a53-b473-bfa84b160cb2,DISK]
2024-12-11T20:12:45,388 WARN  [Thread-975 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741883_1066
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:46049
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:45,388 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1278250962_22 at /127.0.0.1:49742 [Receiving block BP-773969328-172.17.0.2-1733947940734:blk_1073741883_1066] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/cluster_f21598ea-49df-c0f8-b759-6a4f22597d28/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/cluster_f21598ea-49df-c0f8-b759-6a4f22597d28/data/data10]'}, localName='127.0.0.1:38463', datanodeUuid='ce0d95cd-8c3d-4ad7-8b80-989a1cd7ec68', xmitsInProgress=0}:Exception transferring block BP-773969328-172.17.0.2-1733947940734:blk_1073741883_1066 to mirror 127.0.0.1:46049
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:12:45,388 WARN  [Thread-975 {}] hdfs.DataStreamer(1731): Error Recovery for BP-773969328-172.17.0.2-1733947940734:blk_1073741883_1066 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38463,DS-b0a6421d-98c9-48af-ab16-a87f2dea8cbb,DISK], DatanodeInfoWithStorage[127.0.0.1:46049,DS-cb434015-7991-4b0a-b64b-1a5c770c280f,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:46049,DS-cb434015-7991-4b0a-b64b-1a5c770c280f,DISK]) is bad.
2024-12-11T20:12:45,388 WARN  [Thread-975 {}] hdfs.DataStreamer(1850): Abandoning BP-773969328-172.17.0.2-1733947940734:blk_1073741883_1066
2024-12-11T20:12:45,388 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1278250962_22 at /127.0.0.1:49742 [Receiving block BP-773969328-172.17.0.2-1733947940734:blk_1073741883_1066] {}] datanode.BlockReceiver(316): Block 1073741883 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-11T20:12:45,388 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1278250962_22 at /127.0.0.1:49742 [Receiving block BP-773969328-172.17.0.2-1733947940734:blk_1073741883_1066] {}] datanode.DataXceiver(331): 127.0.0.1:38463:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:49742 dst: /127.0.0.1:38463
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:12:45,389 WARN  [Thread-975 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46049,DS-cb434015-7991-4b0a-b64b-1a5c770c280f,DISK]
2024-12-11T20:12:45,390 WARN  [Thread-975 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741884_1067
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:45,390 WARN  [Thread-975 {}] hdfs.DataStreamer(1731): Error Recovery for BP-773969328-172.17.0.2-1733947940734:blk_1073741884_1067 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK], DatanodeInfoWithStorage[127.0.0.1:38463,DS-b0a6421d-98c9-48af-ab16-a87f2dea8cbb,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]) is bad.
2024-12-11T20:12:45,390 WARN  [Thread-975 {}] hdfs.DataStreamer(1850): Abandoning BP-773969328-172.17.0.2-1733947940734:blk_1073741884_1067
2024-12-11T20:12:45,390 WARN  [Thread-975 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]
2024-12-11T20:12:45,391 WARN  [Thread-975 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741885_1068
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:45,391 WARN  [Thread-975 {}] hdfs.DataStreamer(1731): Error Recovery for BP-773969328-172.17.0.2-1733947940734:blk_1073741885_1068 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35127,DS-20198564-3dc6-402d-9d4b-7023e3d1e57b,DISK], DatanodeInfoWithStorage[127.0.0.1:38463,DS-b0a6421d-98c9-48af-ab16-a87f2dea8cbb,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35127,DS-20198564-3dc6-402d-9d4b-7023e3d1e57b,DISK]) is bad.
2024-12-11T20:12:45,391 WARN  [Thread-975 {}] hdfs.DataStreamer(1850): Abandoning BP-773969328-172.17.0.2-1733947940734:blk_1073741885_1068
2024-12-11T20:12:45,392 WARN  [Thread-975 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35127,DS-20198564-3dc6-402d-9d4b-7023e3d1e57b,DISK]
2024-12-11T20:12:45,392 WARN  [IPC Server handler 2 on default port 40619 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-11T20:12:45,392 WARN  [IPC Server handler 2 on default port 40619 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-11T20:12:45,393 WARN  [IPC Server handler 2 on default port 40619 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-11T20:12:45,396 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38463 is added to blk_1073741886_1069 (size=18097)
2024-12-11T20:12:45,804 DEBUG [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/.tmp/info/7aaef61b9f1e4df8a6a16ff64526d42f as hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/info/7aaef61b9f1e4df8a6a16ff64526d42f
2024-12-11T20:12:45,811 INFO  [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5109ba8daed9e21fd7c045352880c060/info of 5109ba8daed9e21fd7c045352880c060 into 7aaef61b9f1e4df8a6a16ff64526d42f(size=17.7 K), total size for store is 17.7 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-11T20:12:45,811 DEBUG [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5109ba8daed9e21fd7c045352880c060:

2024-12-11T20:12:45,811 INFO  [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1733947943302.5109ba8daed9e21fd7c045352880c060., storeName=5109ba8daed9e21fd7c045352880c060/info, priority=13, startTime=1733947965365; duration=0sec
2024-12-11T20:12:45,811 DEBUG [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K
2024-12-11T20:12:45,811 DEBUG [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-11T20:12:45,811 DEBUG [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/info/7aaef61b9f1e4df8a6a16ff64526d42f because midkey is the same as first or last row
2024-12-11T20:12:45,811 DEBUG [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K
2024-12-11T20:12:45,811 DEBUG [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-11T20:12:45,811 DEBUG [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/info/7aaef61b9f1e4df8a6a16ff64526d42f because midkey is the same as first or last row
2024-12-11T20:12:45,811 DEBUG [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K
2024-12-11T20:12:45,811 DEBUG [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-11T20:12:45,812 DEBUG [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/info/7aaef61b9f1e4df8a6a16ff64526d42f because midkey is the same as first or last row
2024-12-11T20:12:45,812 DEBUG [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-11T20:12:45,812 DEBUG [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5109ba8daed9e21fd7c045352880c060:info
2024-12-11T20:12:45,971 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@7ab0f4af[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:38463, datanodeUuid=ce0d95cd-8c3d-4ad7-8b80-989a1cd7ec68, infoPort=41537, infoSecurePort=0, ipcPort=42607, storageInfo=lv=-57;cid=testClusterID;nsid=1349632812;c=1733947940734):Failed to transfer BP-773969328-172.17.0.2-1733947940734:blk_1073741841_1024 to 127.0.0.1:36207 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:12:45,971 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@74770451[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:38463, datanodeUuid=ce0d95cd-8c3d-4ad7-8b80-989a1cd7ec68, infoPort=41537, infoSecurePort=0, ipcPort=42607, storageInfo=lv=-57;cid=testClusterID;nsid=1349632812;c=1733947940734):Failed to transfer BP-773969328-172.17.0.2-1733947940734:blk_1073741861_1044 to 127.0.0.1:35127 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:12:46,070 WARN  [regionserver/9fe0640122ec:0.logRoller {}] wal.FSHLog(539): Too many consecutive RollWriter requests, it's a sign of the total number of live datanodes is lower than the tolerable replicas.
2024-12-11T20:12:46,070 INFO  [regionserver/9fe0640122ec:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:46,119 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:46,159 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T20:12:46,162 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-11T20:12:46,163 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-11T20:12:46,163 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-11T20:12:46,163 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-11T20:12:46,164 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@70a14b1c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/hadoop.log.dir/,AVAILABLE}
2024-12-11T20:12:46,164 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@413b5d87{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-11T20:12:46,280 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3f2f9a23{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/java.io.tmpdir/jetty-localhost-40501-hadoop-hdfs-3_4_1-tests_jar-_-any-16051720533998843756/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T20:12:46,280 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4373c46e{HTTP/1.1, (http/1.1)}{localhost:40501}
2024-12-11T20:12:46,281 INFO  [Time-limited test {}] server.Server(415): Started @129876ms
2024-12-11T20:12:46,282 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-11T20:12:46,382 WARN  [Thread-994 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-11T20:12:46,392 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4534c4613efd666d with lease ID 0x66af004492a15cce: from storage DS-a9e47326-b9f8-4a53-b473-bfa84b160cb2 node DatanodeRegistration(127.0.0.1:37255, datanodeUuid=669f8eed-2f96-4854-8c9a-69c5c7a3d0c2, infoPort=37019, infoSecurePort=0, ipcPort=33699, storageInfo=lv=-57;cid=testClusterID;nsid=1349632812;c=1733947940734), blocks: 6, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-11T20:12:46,392 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4534c4613efd666d with lease ID 0x66af004492a15cce: from storage DS-4f4193c3-6231-4e76-9dbd-6338e3774ec4 node DatanodeRegistration(127.0.0.1:37255, datanodeUuid=669f8eed-2f96-4854-8c9a-69c5c7a3d0c2, infoPort=37019, infoSecurePort=0, ipcPort=33699, storageInfo=lv=-57;cid=testClusterID;nsid=1349632812;c=1733947940734), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T20:12:46,970 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@74770451[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:38463, datanodeUuid=ce0d95cd-8c3d-4ad7-8b80-989a1cd7ec68, infoPort=41537, infoSecurePort=0, ipcPort=42607, storageInfo=lv=-57;cid=testClusterID;nsid=1349632812;c=1733947940734):Failed to transfer BP-773969328-172.17.0.2-1733947940734:blk_1073741866_1049 to 127.0.0.1:46049 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:12:46,970 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@7ab0f4af[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:38463, datanodeUuid=ce0d95cd-8c3d-4ad7-8b80-989a1cd7ec68, infoPort=41537, infoSecurePort=0, ipcPort=42607, storageInfo=lv=-57;cid=testClusterID;nsid=1349632812;c=1733947940734):Failed to transfer BP-773969328-172.17.0.2-1733947940734:blk_1073741871_1054 to 127.0.0.1:35127 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:12:47,254 INFO  [regionserver/9fe0640122ec:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:48,070 INFO  [regionserver/9fe0640122ec:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:48,119 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:48,971 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@7ab0f4af[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:38463, datanodeUuid=ce0d95cd-8c3d-4ad7-8b80-989a1cd7ec68, infoPort=41537, infoSecurePort=0, ipcPort=42607, storageInfo=lv=-57;cid=testClusterID;nsid=1349632812;c=1733947940734):Failed to transfer BP-773969328-172.17.0.2-1733947940734:blk_1073741856_1039 to 127.0.0.1:35127 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:12:48,974 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37255 is added to blk_1073741881_1064 (size=6027)
2024-12-11T20:12:49,254 INFO  [regionserver/9fe0640122ec:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:49,972 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@74770451[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:38463, datanodeUuid=ce0d95cd-8c3d-4ad7-8b80-989a1cd7ec68, infoPort=41537, infoSecurePort=0, ipcPort=42607, storageInfo=lv=-57;cid=testClusterID;nsid=1349632812;c=1733947940734):Failed to transfer BP-773969328-172.17.0.2-1733947940734:blk_1073741886_1069 to 127.0.0.1:46049 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:12:50,070 INFO  [regionserver/9fe0640122ec:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:50,120 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:51,254 INFO  [regionserver/9fe0640122ec:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:51,526 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-11T20:12:52,071 INFO  [regionserver/9fe0640122ec:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:52,120 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:52,275 ERROR [FSHLog-0-hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/MasterData-prefix:9fe0640122ec,38011,1733947941956 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:52,275 WARN  [FSHLog-0-hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/MasterData-prefix:9fe0640122ec,38011,1733947941956 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:52,275 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL FSHLog 9fe0640122ec%2C38011%2C1733947941956:(num 1733947942154) roll requested
2024-12-11T20:12:52,275 INFO  [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9fe0640122ec%2C38011%2C1733947941956.1733947972275
2024-12-11T20:12:52,285 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:12:52,285 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:12:52,285 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:12:52,285 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:12:52,286 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:12:52,286 INFO  [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/MasterData/WALs/9fe0640122ec,38011,1733947941956/9fe0640122ec%2C38011%2C1733947941956.1733947942154 with entries=54, filesize=26.67 KB; new WAL /user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/MasterData/WALs/9fe0640122ec,38011,1733947941956/9fe0640122ec%2C38011%2C1733947941956.1733947972275
2024-12-11T20:12:52,286 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:52,286 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:52,287 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/MasterData/WALs/9fe0640122ec,38011,1733947941956/9fe0640122ec%2C38011%2C1733947941956.1733947942154
2024-12-11T20:12:52,287 WARN  [IPC Server handler 3 on default port 40619 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/MasterData/WALs/9fe0640122ec,38011,1733947941956/9fe0640122ec%2C38011%2C1733947941956.1733947942154 has not been closed. Lease recovery is in progress. RecoveryId = 1071 for block blk_1073741830_1006
2024-12-11T20:12:52,287 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/MasterData/WALs/9fe0640122ec,38011,1733947941956/9fe0640122ec%2C38011%2C1733947941956.1733947942154 after 0ms
2024-12-11T20:12:52,289 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41537:41537),(127.0.0.1/127.0.0.1:37019:37019)]
2024-12-11T20:12:52,289 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/MasterData/WALs/9fe0640122ec,38011,1733947941956/9fe0640122ec%2C38011%2C1733947941956.1733947942154 is not closed yet, will try archiving it next time
2024-12-11T20:12:53,255 INFO  [regionserver/9fe0640122ec:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:54,071 INFO  [regionserver/9fe0640122ec:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:55,255 INFO  [regionserver/9fe0640122ec:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:56,071 INFO  [regionserver/9fe0640122ec:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:56,289 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/MasterData/WALs/9fe0640122ec,38011,1733947941956/9fe0640122ec%2C38011%2C1733947941956.1733947942154 after 4002ms
2024-12-11T20:12:56,409 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@40e8d1f8 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-773969328-172.17.0.2-1733947940734:blk_1073741833_1009, datanode=DatanodeInfoWithStorage[127.0.0.1:34893,null,null])
java.net.ConnectException: Call From 9fe0640122ec/172.17.0.2 to localhost:41499 failed on connection exception: java.net.ConnectException: Connection refused; For more details see:  http://wiki.apache.org/hadoop/ConnectionRefused
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-12-11T20:12:56,411 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37255 is added to blk_1073741833_1020 (size=455)
2024-12-11T20:12:57,041 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.1733947942541 to hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/oldWALs/9fe0640122ec%2C42593%2C1733947942024.1733947942541
2024-12-11T20:12:57,042 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.1733947962034 to hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/oldWALs/9fe0640122ec%2C42593%2C1733947942024.1733947962034
2024-12-11T20:12:57,256 INFO  [regionserver/9fe0640122ec:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:57,387 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@716296d0[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:37255, datanodeUuid=669f8eed-2f96-4854-8c9a-69c5c7a3d0c2, infoPort=37019, infoSecurePort=0, ipcPort=33699, storageInfo=lv=-57;cid=testClusterID;nsid=1349632812;c=1733947940734):Failed to transfer BP-773969328-172.17.0.2-1733947940734:blk_1073741835_1011 to 127.0.0.1:46049 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:12:57,389 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38463 is added to blk_1073741831_1007 (size=1321)
2024-12-11T20:12:58,072 INFO  [regionserver/9fe0640122ec:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:58,387 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@26264c9a[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:37255, datanodeUuid=669f8eed-2f96-4854-8c9a-69c5c7a3d0c2, infoPort=37019, infoSecurePort=0, ipcPort=33699, storageInfo=lv=-57;cid=testClusterID;nsid=1349632812;c=1733947940734):Failed to transfer BP-773969328-172.17.0.2-1733947940734:blk_1073741829_1005 to 127.0.0.1:46049 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:12:58,387 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@716296d0[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:37255, datanodeUuid=669f8eed-2f96-4854-8c9a-69c5c7a3d0c2, infoPort=37019, infoSecurePort=0, ipcPort=33699, storageInfo=lv=-57;cid=testClusterID;nsid=1349632812;c=1733947940734):Failed to transfer BP-773969328-172.17.0.2-1733947940734:blk_1073741827_1003 to 127.0.0.1:46049 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:12:59,256 INFO  [regionserver/9fe0640122ec:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:59,813 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9fe0640122ec%2C42593%2C1733947942024.1733947979812
2024-12-11T20:12:59,816 WARN  [Thread-1027 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741888_1072
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:12:59,816 WARN  [Thread-1027 {}] hdfs.DataStreamer(1731): Error Recovery for BP-773969328-172.17.0.2-1733947940734:blk_1073741888_1072 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46049,DS-cb434015-7991-4b0a-b64b-1a5c770c280f,DISK], DatanodeInfoWithStorage[127.0.0.1:37255,DS-a9e47326-b9f8-4a53-b473-bfa84b160cb2,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:46049,DS-cb434015-7991-4b0a-b64b-1a5c770c280f,DISK]) is bad.
2024-12-11T20:12:59,816 WARN  [Thread-1027 {}] hdfs.DataStreamer(1850): Abandoning BP-773969328-172.17.0.2-1733947940734:blk_1073741888_1072
2024-12-11T20:12:59,817 WARN  [Thread-1027 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46049,DS-cb434015-7991-4b0a-b64b-1a5c770c280f,DISK]
2024-12-11T20:12:59,821 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:12:59,821 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:12:59,821 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:12:59,821 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:12:59,821 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:12:59,821 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.1733947964054 with entries=13, filesize=12.60 KB; new WAL /user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.1733947979812
2024-12-11T20:12:59,822 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37019:37019),(127.0.0.1/127.0.0.1:41537:41537)]
2024-12-11T20:12:59,822 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.1733947964054 is not closed yet, will try archiving it next time
2024-12-11T20:12:59,823 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38463 is added to blk_1073741876_1059 (size=12911)
2024-12-11T20:12:59,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42593 {}] regionserver.HRegion(8855): Flush requested on 5109ba8daed9e21fd7c045352880c060
2024-12-11T20:12:59,827 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5109ba8daed9e21fd7c045352880c060 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-12-11T20:12:59,832 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/.tmp/info/dc1582b72404411c9a98d623387e0d18 is 1080, key is row0013/info:/1733947979824/Put/seqid=0
2024-12-11T20:12:59,838 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38463 is added to blk_1073741890_1074 (size=8190)
2024-12-11T20:12:59,838 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37255 is added to blk_1073741890_1074 (size=8190)
2024-12-11T20:12:59,839 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/.tmp/info/dc1582b72404411c9a98d623387e0d18
2024-12-11T20:12:59,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/.tmp/info/dc1582b72404411c9a98d623387e0d18 as hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/info/dc1582b72404411c9a98d623387e0d18
2024-12-11T20:12:59,852 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/info/dc1582b72404411c9a98d623387e0d18, entries=3, sequenceid=66, filesize=8.0 K
2024-12-11T20:12:59,853 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7527, heapSize ~8.11 KB/8304, currentSize=9.46 KB/9683 for 5109ba8daed9e21fd7c045352880c060 in 26ms, sequenceid=66, compaction requested=false
2024-12-11T20:12:59,853 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5109ba8daed9e21fd7c045352880c060:

2024-12-11T20:12:59,853 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=25.7 K, sizeToCheck=16.0 K
2024-12-11T20:12:59,853 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-11T20:12:59,853 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/info/7aaef61b9f1e4df8a6a16ff64526d42f because midkey is the same as first or last row
2024-12-11T20:13:00,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42593 {}] regionserver.HRegion(8855): Flush requested on 5109ba8daed9e21fd7c045352880c060
2024-12-11T20:13:00,048 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5109ba8daed9e21fd7c045352880c060 1/1 column families, dataSize=10.51 KB heapSize=11.50 KB
2024-12-11T20:13:00,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/.tmp/info/2da2bdf2d21d468da06758db122363fd is 1080, key is row0015/info:/1733947979828/Put/seqid=0
2024-12-11T20:13:00,059 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38463 is added to blk_1073741891_1075 (size=14660)
2024-12-11T20:13:00,059 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37255 is added to blk_1073741891_1075 (size=14660)
2024-12-11T20:13:00,060 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.51 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/.tmp/info/2da2bdf2d21d468da06758db122363fd
2024-12-11T20:13:00,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/.tmp/info/2da2bdf2d21d468da06758db122363fd as hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/info/2da2bdf2d21d468da06758db122363fd
2024-12-11T20:13:00,072 INFO  [regionserver/9fe0640122ec:0.logRoller {}] wal.FSHLog(556): LowReplication-Roller was enabled.
2024-12-11T20:13:00,072 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/info/2da2bdf2d21d468da06758db122363fd, entries=9, sequenceid=79, filesize=14.3 K
2024-12-11T20:13:00,072 INFO  [regionserver/9fe0640122ec:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:13:00,074 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.51 KB/10758, heapSize ~11.48 KB/11760, currentSize=0 B/0 for 5109ba8daed9e21fd7c045352880c060 in 25ms, sequenceid=79, compaction requested=true
2024-12-11T20:13:00,074 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5109ba8daed9e21fd7c045352880c060:

2024-12-11T20:13:00,074 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=40.0 K, sizeToCheck=16.0 K
2024-12-11T20:13:00,074 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-11T20:13:00,074 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/info/7aaef61b9f1e4df8a6a16ff64526d42f because midkey is the same as first or last row
2024-12-11T20:13:00,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5109ba8daed9e21fd7c045352880c060:info, priority=-2147483648, current under compaction store size is 1
2024-12-11T20:13:00,074 DEBUG [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-11T20:13:00,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-11T20:13:00,075 DEBUG [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 40947 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-11T20:13:00,075 DEBUG [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] regionserver.HStore(1541): 5109ba8daed9e21fd7c045352880c060/info is initiating minor compaction (all files)
2024-12-11T20:13:00,075 INFO  [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5109ba8daed9e21fd7c045352880c060/info in TestLogRolling-testLogRollOnDatanodeDeath,,1733947943302.5109ba8daed9e21fd7c045352880c060.
2024-12-11T20:13:00,076 INFO  [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/info/7aaef61b9f1e4df8a6a16ff64526d42f, hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/info/dc1582b72404411c9a98d623387e0d18, hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/info/2da2bdf2d21d468da06758db122363fd] into tmpdir=hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/.tmp, totalSize=40.0 K
2024-12-11T20:13:00,076 DEBUG [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7aaef61b9f1e4df8a6a16ff64526d42f, keycount=12, bloomtype=ROW, size=17.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1733947956025
2024-12-11T20:13:00,076 DEBUG [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc1582b72404411c9a98d623387e0d18, keycount=3, bloomtype=ROW, size=8.0 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1733947965946
2024-12-11T20:13:00,077 DEBUG [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2da2bdf2d21d468da06758db122363fd, keycount=9, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1733947979828
2024-12-11T20:13:00,088 INFO  [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5109ba8daed9e21fd7c045352880c060#info#compaction#27 average throughput is 11.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-11T20:13:00,089 DEBUG [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/.tmp/info/343c4340fc364cbdb1fb426c32561822 is 1080, key is row0002/info:/1733947956025/Put/seqid=0
2024-12-11T20:13:00,093 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37255 is added to blk_1073741892_1076 (size=28989)
2024-12-11T20:13:00,094 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38463 is added to blk_1073741892_1076 (size=28989)
2024-12-11T20:13:00,100 DEBUG [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/.tmp/info/343c4340fc364cbdb1fb426c32561822 as hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/info/343c4340fc364cbdb1fb426c32561822
2024-12-11T20:13:00,106 INFO  [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5109ba8daed9e21fd7c045352880c060/info of 5109ba8daed9e21fd7c045352880c060 into 343c4340fc364cbdb1fb426c32561822(size=28.3 K), total size for store is 28.3 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-11T20:13:00,106 DEBUG [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5109ba8daed9e21fd7c045352880c060:

2024-12-11T20:13:00,106 INFO  [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1733947943302.5109ba8daed9e21fd7c045352880c060., storeName=5109ba8daed9e21fd7c045352880c060/info, priority=13, startTime=1733947980074; duration=0sec
2024-12-11T20:13:00,106 DEBUG [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.3 K, sizeToCheck=16.0 K
2024-12-11T20:13:00,106 DEBUG [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-11T20:13:00,107 DEBUG [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/info/343c4340fc364cbdb1fb426c32561822 because midkey is the same as first or last row
2024-12-11T20:13:00,107 DEBUG [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.3 K, sizeToCheck=16.0 K
2024-12-11T20:13:00,107 DEBUG [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-11T20:13:00,107 DEBUG [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/info/343c4340fc364cbdb1fb426c32561822 because midkey is the same as first or last row
2024-12-11T20:13:00,107 DEBUG [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.3 K, sizeToCheck=16.0 K
2024-12-11T20:13:00,107 DEBUG [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-11T20:13:00,107 DEBUG [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/info/343c4340fc364cbdb1fb426c32561822 because midkey is the same as first or last row
2024-12-11T20:13:00,107 DEBUG [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-11T20:13:00,107 DEBUG [RS:0;9fe0640122ec:42593-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5109ba8daed9e21fd7c045352880c060:info
2024-12-11T20:13:00,224 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.1733947964054 to hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/oldWALs/9fe0640122ec%2C42593%2C1733947942024.1733947964054
2024-12-11T20:13:00,249 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-12-11T20:13:00,249 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-11T20:13:00,249 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-11T20:13:00,249 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T20:13:00,249 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T20:13:00,249 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-12-11T20:13:00,249 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-11T20:13:00,249 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=97545088, stopped=false
2024-12-11T20:13:00,250 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=9fe0640122ec,38011,1733947941956
2024-12-11T20:13:00,251 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42593-0x100caf7f5180001, quorum=127.0.0.1:56424, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-11T20:13:00,251 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42419-0x100caf7f5180002, quorum=127.0.0.1:56424, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-11T20:13:00,251 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38011-0x100caf7f5180000, quorum=127.0.0.1:56424, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-11T20:13:00,251 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42419-0x100caf7f5180002, quorum=127.0.0.1:56424, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:13:00,251 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42593-0x100caf7f5180001, quorum=127.0.0.1:56424, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:13:00,251 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-11T20:13:00,252 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38011-0x100caf7f5180000, quorum=127.0.0.1:56424, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:13:00,252 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-11T20:13:00,252 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-11T20:13:00,252 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T20:13:00,252 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9fe0640122ec,42593,1733947942024' *****
2024-12-11T20:13:00,252 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-12-11T20:13:00,252 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9fe0640122ec,42419,1733947943179' *****
2024-12-11T20:13:00,252 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-12-11T20:13:00,252 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42419-0x100caf7f5180002, quorum=127.0.0.1:56424, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-11T20:13:00,252 INFO  [RS:0;9fe0640122ec:42593 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-11T20:13:00,253 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42593-0x100caf7f5180001, quorum=127.0.0.1:56424, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-11T20:13:00,253 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38011-0x100caf7f5180000, quorum=127.0.0.1:56424, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-11T20:13:00,253 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-12-11T20:13:00,253 INFO  [RS:0;9fe0640122ec:42593 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-11T20:13:00,253 INFO  [RS:1;9fe0640122ec:42419 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-11T20:13:00,253 INFO  [RS:0;9fe0640122ec:42593 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-11T20:13:00,253 INFO  [RS:1;9fe0640122ec:42419 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-11T20:13:00,253 INFO  [RS:1;9fe0640122ec:42419 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-11T20:13:00,253 INFO  [RS:1;9fe0640122ec:42419 {}] regionserver.HRegionServer(959): stopping server 9fe0640122ec,42419,1733947943179
2024-12-11T20:13:00,253 INFO  [RS:1;9fe0640122ec:42419 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-11T20:13:00,253 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-12-11T20:13:00,253 INFO  [RS:0;9fe0640122ec:42593 {}] regionserver.HRegionServer(3091): Received CLOSE for 5109ba8daed9e21fd7c045352880c060
2024-12-11T20:13:00,253 INFO  [RS:1;9fe0640122ec:42419 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;9fe0640122ec:42419.
2024-12-11T20:13:00,253 DEBUG [RS:1;9fe0640122ec:42419 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-11T20:13:00,253 DEBUG [RS:1;9fe0640122ec:42419 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T20:13:00,253 INFO  [RS:1;9fe0640122ec:42419 {}] regionserver.HRegionServer(976): stopping server 9fe0640122ec,42419,1733947943179; all regions closed.
2024-12-11T20:13:00,254 INFO  [RS:0;9fe0640122ec:42593 {}] regionserver.HRegionServer(959): stopping server 9fe0640122ec,42593,1733947942024
2024-12-11T20:13:00,254 INFO  [RS:0;9fe0640122ec:42593 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-11T20:13:00,254 INFO  [RS:0;9fe0640122ec:42593 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;9fe0640122ec:42593.
2024-12-11T20:13:00,254 DEBUG [RS:0;9fe0640122ec:42593 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-11T20:13:00,254 DEBUG [RS:0;9fe0640122ec:42593 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T20:13:00,254 INFO  [RS:0;9fe0640122ec:42593 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-11T20:13:00,254 INFO  [RS:0;9fe0640122ec:42593 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-11T20:13:00,254 INFO  [RS:0;9fe0640122ec:42593 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-11T20:13:00,254 INFO  [RS:0;9fe0640122ec:42593 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-12-11T20:13:00,254 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5109ba8daed9e21fd7c045352880c060, disabling compactions & flushes
2024-12-11T20:13:00,254 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:00,254 INFO  [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1733947943302.5109ba8daed9e21fd7c045352880c060.
2024-12-11T20:13:00,254 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:00,254 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1733947943302.5109ba8daed9e21fd7c045352880c060.
2024-12-11T20:13:00,254 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1733947943302.5109ba8daed9e21fd7c045352880c060. after waiting 0 ms
2024-12-11T20:13:00,254 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1733947943302.5109ba8daed9e21fd7c045352880c060.
2024-12-11T20:13:00,254 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:00,254 INFO  [RS:0;9fe0640122ec:42593 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-12-11T20:13:00,254 DEBUG [RS:0;9fe0640122ec:42593 {}] regionserver.HRegionServer(1325): Online Regions={5109ba8daed9e21fd7c045352880c060=TestLogRolling-testLogRollOnDatanodeDeath,,1733947943302.5109ba8daed9e21fd7c045352880c060., 1588230740=hbase:meta,,1.1588230740}
2024-12-11T20:13:00,254 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:00,254 DEBUG [RS:0;9fe0640122ec:42593 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 5109ba8daed9e21fd7c045352880c060
2024-12-11T20:13:00,254 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:00,254 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-11T20:13:00,255 INFO  [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-11T20:13:00,255 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-11T20:13:00,255 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-11T20:13:00,255 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-11T20:13:00,255 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733947943302.5109ba8daed9e21fd7c045352880c060.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/info/c715426bd5aa4aa0af6e97e1bd20c1f1, hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/info/f4335984ea3a439692479308d3f53619, hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/info/959cda3dfb4b406f8b936903be745d84, hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/info/702cafe2606e4ce29c0bc91325d7d2b6, hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/info/3577146a7f4e49b787498369a919c703, hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/info/7aaef61b9f1e4df8a6a16ff64526d42f, hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/info/ac10ce55978746efa451b84291a89ce9, hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/info/dc1582b72404411c9a98d623387e0d18, hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/info/2da2bdf2d21d468da06758db122363fd] to archive
2024-12-11T20:13:00,255 INFO  [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.71 KB heapSize=3.75 KB
2024-12-11T20:13:00,255 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:13:00,255 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:13:00,255 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
2024-12-11T20:13:00,255 ERROR [FSHLog-0-hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f-prefix:9fe0640122ec,42593,1733947942024.meta {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:13:00,255 WARN  [FSHLog-0-hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f-prefix:9fe0640122ec,42593,1733947942024.meta {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:13:00,256 DEBUG [regionserver/9fe0640122ec:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 9fe0640122ec%2C42593%2C1733947942024.meta:.meta(num 1733947942936) roll requested
2024-12-11T20:13:00,256 INFO  [regionserver/9fe0640122ec:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9fe0640122ec%2C42593%2C1733947942024.meta.1733947980256.meta
2024-12-11T20:13:00,256 WARN  [IPC Server handler 0 on default port 40619 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403 has not been closed. Lease recovery is in progress. RecoveryId = 1077 for block blk_1073741837_1013
2024-12-11T20:13:00,256 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403 after 1ms
2024-12-11T20:13:00,256 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733947943302.5109ba8daed9e21fd7c045352880c060.-1 {}] backup.HFileArchiver(363): Archiving compacted files.
2024-12-11T20:13:00,258 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/info/c715426bd5aa4aa0af6e97e1bd20c1f1 to hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/info/c715426bd5aa4aa0af6e97e1bd20c1f1
2024-12-11T20:13:00,261 WARN  [Thread-1054 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741893_1078
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:46049
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:13:00,261 WARN  [Thread-1054 {}] hdfs.DataStreamer(1731): Error Recovery for BP-773969328-172.17.0.2-1733947940734:blk_1073741893_1078 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38463,DS-b0a6421d-98c9-48af-ab16-a87f2dea8cbb,DISK], DatanodeInfoWithStorage[127.0.0.1:46049,DS-cb434015-7991-4b0a-b64b-1a5c770c280f,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:46049,DS-cb434015-7991-4b0a-b64b-1a5c770c280f,DISK]) is bad.
2024-12-11T20:13:00,261 WARN  [Thread-1054 {}] hdfs.DataStreamer(1850): Abandoning BP-773969328-172.17.0.2-1733947940734:blk_1073741893_1078
2024-12-11T20:13:00,260 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1278250962_22 at /127.0.0.1:48152 [Receiving block BP-773969328-172.17.0.2-1733947940734:blk_1073741893_1078] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/cluster_f21598ea-49df-c0f8-b759-6a4f22597d28/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/cluster_f21598ea-49df-c0f8-b759-6a4f22597d28/data/data10]'}, localName='127.0.0.1:38463', datanodeUuid='ce0d95cd-8c3d-4ad7-8b80-989a1cd7ec68', xmitsInProgress=0}:Exception transferring block BP-773969328-172.17.0.2-1733947940734:blk_1073741893_1078 to mirror 127.0.0.1:46049
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:13:00,261 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-1278250962_22 at /127.0.0.1:48152 [Receiving block BP-773969328-172.17.0.2-1733947940734:blk_1073741893_1078] {}] datanode.BlockReceiver(316): Block 1073741893 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-12-11T20:13:00,261 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1278250962_22 at /127.0.0.1:48152 [Receiving block BP-773969328-172.17.0.2-1733947940734:blk_1073741893_1078] {}] datanode.DataXceiver(331): 127.0.0.1:38463:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:48152 dst: /127.0.0.1:38463
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:13:00,261 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/info/f4335984ea3a439692479308d3f53619 to hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/info/f4335984ea3a439692479308d3f53619
2024-12-11T20:13:00,262 WARN  [Thread-1054 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46049,DS-cb434015-7991-4b0a-b64b-1a5c770c280f,DISK]
2024-12-11T20:13:00,262 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/info/959cda3dfb4b406f8b936903be745d84 to hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/info/959cda3dfb4b406f8b936903be745d84
2024-12-11T20:13:00,262 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/info/702cafe2606e4ce29c0bc91325d7d2b6 to hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/info/702cafe2606e4ce29c0bc91325d7d2b6
2024-12-11T20:13:00,262 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/info/3577146a7f4e49b787498369a919c703 to hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/info/3577146a7f4e49b787498369a919c703
2024-12-11T20:13:00,263 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/info/7aaef61b9f1e4df8a6a16ff64526d42f to hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/info/7aaef61b9f1e4df8a6a16ff64526d42f
2024-12-11T20:13:00,263 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/info/ac10ce55978746efa451b84291a89ce9 to hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/info/ac10ce55978746efa451b84291a89ce9
2024-12-11T20:13:00,264 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/info/dc1582b72404411c9a98d623387e0d18 to hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/info/dc1582b72404411c9a98d623387e0d18
2024-12-11T20:13:00,266 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/info/2da2bdf2d21d468da06758db122363fd to hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/info/2da2bdf2d21d468da06758db122363fd
2024-12-11T20:13:00,266 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:00,266 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:00,266 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:00,266 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:00,266 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:00,266 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733947943302.5109ba8daed9e21fd7c045352880c060.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=9fe0640122ec:38011 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 16 more
2024-12-11T20:13:00,267 WARN  [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733947943302.5109ba8daed9e21fd7c045352880c060.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [c715426bd5aa4aa0af6e97e1bd20c1f1=10347, f4335984ea3a439692479308d3f53619=12506, 959cda3dfb4b406f8b936903be745d84=17994, 702cafe2606e4ce29c0bc91325d7d2b6=6027, 3577146a7f4e49b787498369a919c703=6027, 7aaef61b9f1e4df8a6a16ff64526d42f=18097, ac10ce55978746efa451b84291a89ce9=6027, dc1582b72404411c9a98d623387e0d18=8190, 2da2bdf2d21d468da06758db122363fd=14660]
2024-12-11T20:13:00,267 INFO  [regionserver/9fe0640122ec:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta with entries=8, filesize=2.33 KB; new WAL /user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947980256.meta
2024-12-11T20:13:00,267 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:13:00,267 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34893,DS-c09ca836-bfa5-489f-bed7-23a82edbbc28,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:13:00,267 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
2024-12-11T20:13:00,268 WARN  [IPC Server handler 3 on default port 40619 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta has not been closed. Lease recovery is in progress. RecoveryId = 1080 for block blk_1073741834_1010
2024-12-11T20:13:00,268 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta after 1ms
2024-12-11T20:13:00,272 DEBUG [regionserver/9fe0640122ec:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37019:37019),(127.0.0.1/127.0.0.1:41537:41537)]
2024-12-11T20:13:00,272 DEBUG [regionserver/9fe0640122ec:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta is not closed yet, will try archiving it next time
2024-12-11T20:13:00,273 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/default/TestLogRolling-testLogRollOnDatanodeDeath/5109ba8daed9e21fd7c045352880c060/recovered.edits/83.seqid, newMaxSeqId=83, maxSeqId=1
2024-12-11T20:13:00,274 INFO  [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1733947943302.5109ba8daed9e21fd7c045352880c060.
2024-12-11T20:13:00,274 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5109ba8daed9e21fd7c045352880c060:
Waiting for close lock at 1733947980254Running coprocessor pre-close hooks at 1733947980254Disabling compacts and flushes for region at 1733947980254Disabling writes for close at 1733947980254Writing region close event to WAL at 1733947980267 (+13 ms)Running coprocessor post-close hooks at 1733947980274 (+7 ms)Closed at 1733947980274
2024-12-11T20:13:00,274 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1733947943302.5109ba8daed9e21fd7c045352880c060.
2024-12-11T20:13:00,291 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/hbase/meta/1588230740/.tmp/info/e2572433822d42478ec034227808160a is 203, key is TestLogRolling-testLogRollOnDatanodeDeath,,1733947943302.5109ba8daed9e21fd7c045352880c060./info:regioninfo/1733947943673/Put/seqid=0
2024-12-11T20:13:00,293 WARN  [Thread-1062 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741895_1081
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:13:00,294 WARN  [Thread-1062 {}] hdfs.DataStreamer(1731): Error Recovery for BP-773969328-172.17.0.2-1733947940734:blk_1073741895_1081 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46049,DS-cb434015-7991-4b0a-b64b-1a5c770c280f,DISK], DatanodeInfoWithStorage[127.0.0.1:38463,DS-b0a6421d-98c9-48af-ab16-a87f2dea8cbb,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:46049,DS-cb434015-7991-4b0a-b64b-1a5c770c280f,DISK]) is bad.
2024-12-11T20:13:00,294 WARN  [Thread-1062 {}] hdfs.DataStreamer(1850): Abandoning BP-773969328-172.17.0.2-1733947940734:blk_1073741895_1081
2024-12-11T20:13:00,294 WARN  [Thread-1062 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46049,DS-cb434015-7991-4b0a-b64b-1a5c770c280f,DISK]
2024-12-11T20:13:00,299 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38463 is added to blk_1073741896_1082 (size=7089)
2024-12-11T20:13:00,299 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37255 is added to blk_1073741896_1082 (size=7089)
2024-12-11T20:13:00,300 INFO  [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.50 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/hbase/meta/1588230740/.tmp/info/e2572433822d42478ec034227808160a
2024-12-11T20:13:00,321 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/hbase/meta/1588230740/.tmp/ns/d75d900919544a358b0a484616e90211 is 43, key is default/ns:d/1733947943032/Put/seqid=0
2024-12-11T20:13:00,323 WARN  [Thread-1068 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741897_1083
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:13:00,323 WARN  [Thread-1068 {}] hdfs.DataStreamer(1731): Error Recovery for BP-773969328-172.17.0.2-1733947940734:blk_1073741897_1083 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46049,DS-cb434015-7991-4b0a-b64b-1a5c770c280f,DISK], DatanodeInfoWithStorage[127.0.0.1:37255,DS-a9e47326-b9f8-4a53-b473-bfa84b160cb2,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:46049,DS-cb434015-7991-4b0a-b64b-1a5c770c280f,DISK]) is bad.
2024-12-11T20:13:00,323 WARN  [Thread-1068 {}] hdfs.DataStreamer(1850): Abandoning BP-773969328-172.17.0.2-1733947940734:blk_1073741897_1083
2024-12-11T20:13:00,324 WARN  [Thread-1068 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46049,DS-cb434015-7991-4b0a-b64b-1a5c770c280f,DISK]
2024-12-11T20:13:00,328 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37255 is added to blk_1073741898_1084 (size=5153)
2024-12-11T20:13:00,328 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38463 is added to blk_1073741898_1084 (size=5153)
2024-12-11T20:13:00,329 INFO  [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/hbase/meta/1588230740/.tmp/ns/d75d900919544a358b0a484616e90211
2024-12-11T20:13:00,331 INFO  [regionserver/9fe0640122ec:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-12-11T20:13:00,331 INFO  [regionserver/9fe0640122ec:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-12-11T20:13:00,350 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/hbase/meta/1588230740/.tmp/table/a475896e8f5b4cd583ab5c5391932527 is 77, key is TestLogRolling-testLogRollOnDatanodeDeath/table:state/1733947943685/Put/seqid=0
2024-12-11T20:13:00,351 WARN  [Thread-1074 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741899_1085
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:13:00,352 WARN  [Thread-1074 {}] hdfs.DataStreamer(1731): Error Recovery for BP-773969328-172.17.0.2-1733947940734:blk_1073741899_1085 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46049,DS-cb434015-7991-4b0a-b64b-1a5c770c280f,DISK], DatanodeInfoWithStorage[127.0.0.1:37255,DS-a9e47326-b9f8-4a53-b473-bfa84b160cb2,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:46049,DS-cb434015-7991-4b0a-b64b-1a5c770c280f,DISK]) is bad.
2024-12-11T20:13:00,352 WARN  [Thread-1074 {}] hdfs.DataStreamer(1850): Abandoning BP-773969328-172.17.0.2-1733947940734:blk_1073741899_1085
2024-12-11T20:13:00,352 WARN  [Thread-1074 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46049,DS-cb434015-7991-4b0a-b64b-1a5c770c280f,DISK]
2024-12-11T20:13:00,356 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37255 is added to blk_1073741900_1086 (size=5424)
2024-12-11T20:13:00,357 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38463 is added to blk_1073741900_1086 (size=5424)
2024-12-11T20:13:00,357 INFO  [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=146 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/hbase/meta/1588230740/.tmp/table/a475896e8f5b4cd583ab5c5391932527
2024-12-11T20:13:00,363 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/hbase/meta/1588230740/.tmp/info/e2572433822d42478ec034227808160a as hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/hbase/meta/1588230740/info/e2572433822d42478ec034227808160a
2024-12-11T20:13:00,369 INFO  [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/hbase/meta/1588230740/info/e2572433822d42478ec034227808160a, entries=10, sequenceid=11, filesize=6.9 K
2024-12-11T20:13:00,370 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/hbase/meta/1588230740/.tmp/ns/d75d900919544a358b0a484616e90211 as hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/hbase/meta/1588230740/ns/d75d900919544a358b0a484616e90211
2024-12-11T20:13:00,375 INFO  [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/hbase/meta/1588230740/ns/d75d900919544a358b0a484616e90211, entries=2, sequenceid=11, filesize=5.0 K
2024-12-11T20:13:00,376 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/hbase/meta/1588230740/.tmp/table/a475896e8f5b4cd583ab5c5391932527 as hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/hbase/meta/1588230740/table/a475896e8f5b4cd583ab5c5391932527
2024-12-11T20:13:00,380 INFO  [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/hbase/meta/1588230740/table/a475896e8f5b4cd583ab5c5391932527, entries=2, sequenceid=11, filesize=5.3 K
2024-12-11T20:13:00,382 INFO  [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.71 KB/1752, heapSize ~3.45 KB/3536, currentSize=0 B/0 for 1588230740 in 127ms, sequenceid=11, compaction requested=false
2024-12-11T20:13:00,386 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-12-11T20:13:00,387 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-11T20:13:00,387 INFO  [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-11T20:13:00,387 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733947980254Running coprocessor pre-close hooks at 1733947980254Disabling compacts and flushes for region at 1733947980254Disabling writes for close at 1733947980255 (+1 ms)Obtaining lock to block concurrent updates at 1733947980255Preparing flush snapshotting stores in 1588230740 at 1733947980255Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1752, getHeapSize=3776, getOffHeapSize=0, getCellsCount=14 at 1733947980255Flushing stores of hbase:meta,,1.1588230740 at 1733947980273 (+18 ms)Flushing 1588230740/info: creating writer at 1733947980273Flushing 1588230740/info: appending metadata at 1733947980291 (+18 ms)Flushing 1588230740/info: closing flushed file at 1733947980291Flushing 1588230740/ns: creating writer at 1733947980306 (+15 ms)Flushing 1588230740/ns: appending metadata at 1733947980321 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1733947980321Flushing 1588230740/table: creating writer at 1733947980334 (+13 ms)Flushing 1588230740/table: appending metadata at 1733947980349 (+15 ms)Flushing 1588230740/table: closing flushed file at 1733947980349Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@683b7b84: reopening flushed file at 1733947980363 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@33498129: reopening flushed file at 1733947980369 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@621a756c: reopening flushed file at 1733947980375 (+6 ms)Finished flush of dataSize ~1.71 KB/1752, heapSize ~3.45 KB/3536, currentSize=0 B/0 for 1588230740 in 127ms, sequenceid=11, compaction requested=false at 1733947980382 (+7 ms)Writing region close event to WAL at 1733947980383 (+1 ms)Running coprocessor post-close hooks at 1733947980387 (+4 ms)Closed at 1733947980387
2024-12-11T20:13:00,387 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-11T20:13:00,398 INFO  [regionserver/9fe0640122ec:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-11T20:13:00,438 INFO  [regionserver/9fe0640122ec:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-12-11T20:13:00,438 INFO  [regionserver/9fe0640122ec:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-12-11T20:13:00,455 INFO  [RS:0;9fe0640122ec:42593 {}] regionserver.HRegionServer(976): stopping server 9fe0640122ec,42593,1733947942024; all regions closed.
2024-12-11T20:13:00,455 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:00,455 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:00,455 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:00,456 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:00,456 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:00,458 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38463 is added to blk_1073741894_1079 (size=825)
2024-12-11T20:13:00,458 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37255 is added to blk_1073741894_1079 (size=825)
2024-12-11T20:13:00,971 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@74770451[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:38463, datanodeUuid=ce0d95cd-8c3d-4ad7-8b80-989a1cd7ec68, infoPort=41537, infoSecurePort=0, ipcPort=42607, storageInfo=lv=-57;cid=testClusterID;nsid=1349632812;c=1733947940734):Failed to transfer BP-773969328-172.17.0.2-1733947940734:blk_1073741876_1059 to 127.0.0.1:46049 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:13:01,254 INFO  [regionserver/9fe0640122ec:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-11T20:13:01,388 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38463 is added to blk_1073741832_1008 (size=32)
2024-12-11T20:13:01,388 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38463 is added to blk_1073741828_1004 (size=1189)
2024-12-11T20:13:02,156 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath
2024-12-11T20:13:02,156 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-11T20:13:02,156 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-12-11T20:13:02,386 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@26264c9a[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:37255, datanodeUuid=669f8eed-2f96-4854-8c9a-69c5c7a3d0c2, infoPort=37019, infoSecurePort=0, ipcPort=33699, storageInfo=lv=-57;cid=testClusterID;nsid=1349632812;c=1733947940734):Failed to transfer BP-773969328-172.17.0.2-1733947940734:blk_1073741833_1020 to 127.0.0.1:46049 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:13:02,386 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@716296d0[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:37255, datanodeUuid=669f8eed-2f96-4854-8c9a-69c5c7a3d0c2, infoPort=37019, infoSecurePort=0, ipcPort=33699, storageInfo=lv=-57;cid=testClusterID;nsid=1349632812;c=1733947940734):Failed to transfer BP-773969328-172.17.0.2-1733947940734:blk_1073741826_1002 to 127.0.0.1:46049 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:13:03,053 INFO  [master/9fe0640122ec:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-12-11T20:13:03,053 INFO  [master/9fe0640122ec:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-12-11T20:13:03,389 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38463 is added to blk_1073741836_1012 (size=76)
2024-12-11T20:13:03,389 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38463 is added to blk_1073741825_1001 (size=7)
2024-12-11T20:13:04,257 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403 after 4002ms
2024-12-11T20:13:04,269 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta after 4002ms
2024-12-11T20:13:05,255 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds"
2024-12-11T20:13:05,257 INFO  [regionserver/9fe0640122ec:0.logRoller {}] wal.FSHLog(580): 
java.nio.channels.ClosedChannelException: null
	at org.apache.hadoop.hdfs.ExceptionLastSeen.throwException4Close(ExceptionLastSeen.java:73) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSOutputStream.checkClosed(DFSOutputStream.java:158) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSOutputStream.getCurrentBlockReplication(DFSOutputStream.java:775) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.client.HdfsDataOutputStream.getCurrentBlockReplication(HdfsDataOutputStream.java:79) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.getLogReplication(FSHLog.java:577) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doCheckLogLowReplication(FSHLog.java:525) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.checkLogLowReplication(AbstractFSWAL.java:2224) ~[classes/:?]
	at org.apache.hadoop.hbase.wal.AbstractWALRoller.checkLowReplication(AbstractWALRoller.java:148) ~[classes/:?]
	at org.apache.hadoop.hbase.wal.AbstractWALRoller.run(AbstractWALRoller.java:176) ~[classes/:?]
2024-12-11T20:13:05,257 DEBUG [RS:1;9fe0640122ec:42419 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/oldWALs
2024-12-11T20:13:05,257 INFO  [RS:1;9fe0640122ec:42419 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 9fe0640122ec%2C42419%2C1733947943179:(num 1733947943403)
2024-12-11T20:13:05,257 DEBUG [RS:1;9fe0640122ec:42419 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T20:13:05,258 INFO  [RS:1;9fe0640122ec:42419 {}] regionserver.LeaseManager(133): Closed leases
2024-12-11T20:13:05,258 INFO  [RS:1;9fe0640122ec:42419 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-11T20:13:05,258 INFO  [RS:1;9fe0640122ec:42419 {}] hbase.ChoreService(370): Chore service for: regionserver/9fe0640122ec:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-12-11T20:13:05,258 INFO  [RS:1;9fe0640122ec:42419 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-11T20:13:05,258 INFO  [RS:1;9fe0640122ec:42419 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-11T20:13:05,258 INFO  [RS:1;9fe0640122ec:42419 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-11T20:13:05,258 INFO  [regionserver/9fe0640122ec:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-11T20:13:05,258 INFO  [RS:1;9fe0640122ec:42419 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-11T20:13:05,258 INFO  [RS:1;9fe0640122ec:42419 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42419
2024-12-11T20:13:05,260 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42419-0x100caf7f5180002, quorum=127.0.0.1:56424, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9fe0640122ec,42419,1733947943179
2024-12-11T20:13:05,260 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38011-0x100caf7f5180000, quorum=127.0.0.1:56424, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-11T20:13:05,260 INFO  [RS:1;9fe0640122ec:42419 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-11T20:13:05,262 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9fe0640122ec,42419,1733947943179]
2024-12-11T20:13:05,263 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1812) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy45.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$isFileClosed$57(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.isFileClosed(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor104.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy46.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor104.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor104.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1810) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:05,263 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9fe0640122ec,42419,1733947943179 already deleted, retry=false
2024-12-11T20:13:05,263 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 9fe0640122ec,42419,1733947943179 expired; onlineServers=1
2024-12-11T20:13:05,275 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:05,286 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:05,286 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:05,286 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:05,287 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:05,287 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:05,293 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:05,293 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:05,362 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42419-0x100caf7f5180002, quorum=127.0.0.1:56424, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-11T20:13:05,362 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42419-0x100caf7f5180002, quorum=127.0.0.1:56424, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-11T20:13:05,362 INFO  [RS:1;9fe0640122ec:42419 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-11T20:13:05,362 INFO  [RS:1;9fe0640122ec:42419 {}] regionserver.HRegionServer(1031): Exiting; stopping=9fe0640122ec,42419,1733947943179; zookeeper connection closed.
2024-12-11T20:13:05,363 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1ac894b0 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1ac894b0
2024-12-11T20:13:05,456 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds"
2024-12-11T20:13:05,461 DEBUG [RS:0;9fe0640122ec:42593 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/oldWALs
2024-12-11T20:13:05,461 INFO  [RS:0;9fe0640122ec:42593 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 9fe0640122ec%2C42593%2C1733947942024.meta:.meta(num 1733947980256)
2024-12-11T20:13:05,461 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:05,462 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:05,462 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:05,462 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:05,462 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:05,465 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38463 is added to blk_1073741889_1073 (size=16308)
2024-12-11T20:13:05,465 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37255 is added to blk_1073741889_1073 (size=16308)
2024-12-11T20:13:05,469 DEBUG [RS:0;9fe0640122ec:42593 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/oldWALs
2024-12-11T20:13:05,469 INFO  [RS:0;9fe0640122ec:42593 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 9fe0640122ec%2C42593%2C1733947942024:(num 1733947979812)
2024-12-11T20:13:05,469 DEBUG [RS:0;9fe0640122ec:42593 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T20:13:05,469 INFO  [RS:0;9fe0640122ec:42593 {}] regionserver.LeaseManager(133): Closed leases
2024-12-11T20:13:05,469 INFO  [RS:0;9fe0640122ec:42593 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-11T20:13:05,470 INFO  [RS:0;9fe0640122ec:42593 {}] hbase.ChoreService(370): Chore service for: regionserver/9fe0640122ec:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-12-11T20:13:05,470 INFO  [RS:0;9fe0640122ec:42593 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-11T20:13:05,470 INFO  [regionserver/9fe0640122ec:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-11T20:13:05,470 INFO  [RS:0;9fe0640122ec:42593 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42593
2024-12-11T20:13:05,472 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42593-0x100caf7f5180001, quorum=127.0.0.1:56424, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9fe0640122ec,42593,1733947942024
2024-12-11T20:13:05,472 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38011-0x100caf7f5180000, quorum=127.0.0.1:56424, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-11T20:13:05,472 INFO  [RS:0;9fe0640122ec:42593 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-11T20:13:05,474 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9fe0640122ec,42593,1733947942024]
2024-12-11T20:13:05,475 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9fe0640122ec,42593,1733947942024 already deleted, retry=false
2024-12-11T20:13:05,475 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 9fe0640122ec,42593,1733947942024 expired; onlineServers=0
2024-12-11T20:13:05,475 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '9fe0640122ec,38011,1733947941956' *****
2024-12-11T20:13:05,475 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-11T20:13:05,475 INFO  [M:0;9fe0640122ec:38011 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-11T20:13:05,475 INFO  [M:0;9fe0640122ec:38011 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-11T20:13:05,476 DEBUG [M:0;9fe0640122ec:38011 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-11T20:13:05,476 DEBUG [M:0;9fe0640122ec:38011 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-11T20:13:05,476 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-11T20:13:05,476 DEBUG [master/9fe0640122ec:0:becomeActiveMaster-HFileCleaner.large.0-1733947942277 {}] cleaner.HFileCleaner(306): Exit Thread[master/9fe0640122ec:0:becomeActiveMaster-HFileCleaner.large.0-1733947942277,5,FailOnTimeoutGroup]
2024-12-11T20:13:05,476 INFO  [M:0;9fe0640122ec:38011 {}] hbase.ChoreService(370): Chore service for: master/9fe0640122ec:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-12-11T20:13:05,476 INFO  [M:0;9fe0640122ec:38011 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-11T20:13:05,476 DEBUG [M:0;9fe0640122ec:38011 {}] master.HMaster(1795): Stopping service threads
2024-12-11T20:13:05,476 INFO  [M:0;9fe0640122ec:38011 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-11T20:13:05,476 INFO  [M:0;9fe0640122ec:38011 {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-11T20:13:05,476 INFO  [M:0;9fe0640122ec:38011 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-11T20:13:05,477 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-11T20:13:05,477 DEBUG [master/9fe0640122ec:0:becomeActiveMaster-HFileCleaner.small.0-1733947942277 {}] cleaner.HFileCleaner(306): Exit Thread[master/9fe0640122ec:0:becomeActiveMaster-HFileCleaner.small.0-1733947942277,5,FailOnTimeoutGroup]
2024-12-11T20:13:05,477 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38011-0x100caf7f5180000, quorum=127.0.0.1:56424, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-11T20:13:05,477 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38011-0x100caf7f5180000, quorum=127.0.0.1:56424, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:13:05,478 DEBUG [M:0;9fe0640122ec:38011 {}] zookeeper.ZKUtil(347): master:38011-0x100caf7f5180000, quorum=127.0.0.1:56424, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-11T20:13:05,478 WARN  [M:0;9fe0640122ec:38011 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-11T20:13:05,478 INFO  [M:0;9fe0640122ec:38011 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/.lastflushedseqids
2024-12-11T20:13:05,496 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38463 is added to blk_1073741901_1087 (size=130)
2024-12-11T20:13:05,496 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37255 is added to blk_1073741901_1087 (size=130)
2024-12-11T20:13:05,574 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42593-0x100caf7f5180001, quorum=127.0.0.1:56424, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-11T20:13:05,574 INFO  [RS:0;9fe0640122ec:42593 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-11T20:13:05,574 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42593-0x100caf7f5180001, quorum=127.0.0.1:56424, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-11T20:13:05,574 INFO  [RS:0;9fe0640122ec:42593 {}] regionserver.HRegionServer(1031): Exiting; stopping=9fe0640122ec,42593,1733947942024; zookeeper connection closed.
2024-12-11T20:13:05,574 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@d31b3d2 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@d31b3d2
2024-12-11T20:13:05,575 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete
2024-12-11T20:13:05,796 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-11T20:13:05,808 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:05,808 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:05,809 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:05,809 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:05,809 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:05,810 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:05,813 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:05,816 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:05,897 INFO  [M:0;9fe0640122ec:38011 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-12-11T20:13:05,897 INFO  [M:0;9fe0640122ec:38011 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-11T20:13:05,897 DEBUG [M:0;9fe0640122ec:38011 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-11T20:13:05,897 INFO  [M:0;9fe0640122ec:38011 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T20:13:05,897 DEBUG [M:0;9fe0640122ec:38011 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T20:13:05,897 DEBUG [M:0;9fe0640122ec:38011 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-11T20:13:05,898 DEBUG [M:0;9fe0640122ec:38011 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T20:13:05,898 INFO  [M:0;9fe0640122ec:38011 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.25 KB heapSize=29.49 KB
2024-12-11T20:13:05,915 DEBUG [M:0;9fe0640122ec:38011 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a80ed90259e64504aa373131596db9e8 is 82, key is hbase:meta,,1/info:regioninfo/1733947943012/Put/seqid=0
2024-12-11T20:13:05,921 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38463 is added to blk_1073741902_1088 (size=5672)
2024-12-11T20:13:05,922 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37255 is added to blk_1073741902_1088 (size=5672)
2024-12-11T20:13:05,922 INFO  [M:0;9fe0640122ec:38011 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a80ed90259e64504aa373131596db9e8
2024-12-11T20:13:05,944 DEBUG [M:0;9fe0640122ec:38011 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/cd4a2611836d4eb1ae6211fb6d05cb65 is 774, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733947943691/Put/seqid=0
2024-12-11T20:13:05,949 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38463 is added to blk_1073741903_1089 (size=6255)
2024-12-11T20:13:05,950 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37255 is added to blk_1073741903_1089 (size=6255)
2024-12-11T20:13:05,950 INFO  [M:0;9fe0640122ec:38011 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.58 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/cd4a2611836d4eb1ae6211fb6d05cb65
2024-12-11T20:13:05,956 INFO  [M:0;9fe0640122ec:38011 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for cd4a2611836d4eb1ae6211fb6d05cb65
2024-12-11T20:13:05,971 DEBUG [M:0;9fe0640122ec:38011 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4f4a137278b64de7bd2651ede563decc is 69, key is 9fe0640122ec,42419,1733947943179/rs:state/1733947943232/Put/seqid=0
2024-12-11T20:13:05,976 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37255 is added to blk_1073741904_1090 (size=5224)
2024-12-11T20:13:05,976 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38463 is added to blk_1073741904_1090 (size=5224)
2024-12-11T20:13:05,977 INFO  [M:0;9fe0640122ec:38011 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4f4a137278b64de7bd2651ede563decc
2024-12-11T20:13:05,998 DEBUG [M:0;9fe0640122ec:38011 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/cb95aec3fa3f4c9bbcee54f17e3cd986 is 52, key is load_balancer_on/state:d/1733947943162/Put/seqid=0
2024-12-11T20:13:06,003 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38463 is added to blk_1073741905_1091 (size=5056)
2024-12-11T20:13:06,004 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37255 is added to blk_1073741905_1091 (size=5056)
2024-12-11T20:13:06,004 INFO  [M:0;9fe0640122ec:38011 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/cb95aec3fa3f4c9bbcee54f17e3cd986
2024-12-11T20:13:06,010 DEBUG [M:0;9fe0640122ec:38011 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a80ed90259e64504aa373131596db9e8 as hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a80ed90259e64504aa373131596db9e8
2024-12-11T20:13:06,016 INFO  [M:0;9fe0640122ec:38011 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a80ed90259e64504aa373131596db9e8, entries=8, sequenceid=60, filesize=5.5 K
2024-12-11T20:13:06,017 DEBUG [M:0;9fe0640122ec:38011 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/cd4a2611836d4eb1ae6211fb6d05cb65 as hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/cd4a2611836d4eb1ae6211fb6d05cb65
2024-12-11T20:13:06,025 INFO  [M:0;9fe0640122ec:38011 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for cd4a2611836d4eb1ae6211fb6d05cb65
2024-12-11T20:13:06,026 INFO  [M:0;9fe0640122ec:38011 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/cd4a2611836d4eb1ae6211fb6d05cb65, entries=6, sequenceid=60, filesize=6.1 K
2024-12-11T20:13:06,028 DEBUG [M:0;9fe0640122ec:38011 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4f4a137278b64de7bd2651ede563decc as hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/4f4a137278b64de7bd2651ede563decc
2024-12-11T20:13:06,036 INFO  [M:0;9fe0640122ec:38011 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/4f4a137278b64de7bd2651ede563decc, entries=2, sequenceid=60, filesize=5.1 K
2024-12-11T20:13:06,037 DEBUG [M:0;9fe0640122ec:38011 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/cb95aec3fa3f4c9bbcee54f17e3cd986 as hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/cb95aec3fa3f4c9bbcee54f17e3cd986
2024-12-11T20:13:06,043 INFO  [M:0;9fe0640122ec:38011 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/cb95aec3fa3f4c9bbcee54f17e3cd986, entries=1, sequenceid=60, filesize=4.9 K
2024-12-11T20:13:06,045 INFO  [M:0;9fe0640122ec:38011 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.25 KB/23805, heapSize ~29.43 KB/30136, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 147ms, sequenceid=60, compaction requested=false
2024-12-11T20:13:06,057 INFO  [M:0;9fe0640122ec:38011 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T20:13:06,057 DEBUG [M:0;9fe0640122ec:38011 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733947985897Disabling compacts and flushes for region at 1733947985897Disabling writes for close at 1733947985897Obtaining lock to block concurrent updates at 1733947985898 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733947985898Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23805, getHeapSize=30136, getOffHeapSize=0, getCellsCount=71 at 1733947985898Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733947985899 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733947985899Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733947985914 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733947985914Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733947985928 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733947985943 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733947985943Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733947985956 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733947985970 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733947985970Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733947985982 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733947985997 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733947985997Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4901841d: reopening flushed file at 1733947986009 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6b448fd7: reopening flushed file at 1733947986016 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@78d21d8e: reopening flushed file at 1733947986026 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6faf6668: reopening flushed file at 1733947986036 (+10 ms)Finished flush of dataSize ~23.25 KB/23805, heapSize ~29.43 KB/30136, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 147ms, sequenceid=60, compaction requested=false at 1733947986045 (+9 ms)Writing region close event to WAL at 1733947986057 (+12 ms)Closed at 1733947986057
2024-12-11T20:13:06,059 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:06,059 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:06,059 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:06,060 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:06,060 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:06,062 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37255 is added to blk_1073741887_1070 (size=1045)
2024-12-11T20:13:06,062 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38463 is added to blk_1073741887_1070 (size=1045)
2024-12-11T20:13:06,264 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:06,270 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:06,413 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@5e465518 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-773969328-172.17.0.2-1733947940734:blk_1073741830_1006, datanode=DatanodeInfoWithStorage[127.0.0.1:34893,null,null])
java.net.ConnectException: Call From 9fe0640122ec/172.17.0.2 to localhost:41499 failed on connection exception: java.net.ConnectException: Connection refused; For more details see:  http://wiki.apache.org/hadoop/ConnectionRefused
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-12-11T20:13:07,264 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:07,270 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:07,297 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/MasterData/WALs/9fe0640122ec,38011,1733947941956/9fe0640122ec%2C38011%2C1733947941956.1733947942154 to hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/MasterData/oldWALs/9fe0640122ec%2C38011%2C1733947941956.1733947942154
2024-12-11T20:13:07,300 INFO  [WAL-Archive-0 {}] region.MasterRegionUtils(50): Moved hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/MasterData/oldWALs/9fe0640122ec%2C38011%2C1733947941956.1733947942154 to hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/oldWALs/9fe0640122ec%2C38011%2C1733947941956.1733947942154$masterlocalwal$
2024-12-11T20:13:07,300 INFO  [M:0;9fe0640122ec:38011 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-12-11T20:13:07,300 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-11T20:13:07,300 INFO  [M:0;9fe0640122ec:38011 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38011
2024-12-11T20:13:07,301 INFO  [M:0;9fe0640122ec:38011 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-11T20:13:07,403 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38011-0x100caf7f5180000, quorum=127.0.0.1:56424, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-11T20:13:07,403 INFO  [M:0;9fe0640122ec:38011 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-11T20:13:07,403 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38011-0x100caf7f5180000, quorum=127.0.0.1:56424, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-11T20:13:07,406 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3f2f9a23{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T20:13:07,406 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4373c46e{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-11T20:13:07,406 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-11T20:13:07,406 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@413b5d87{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-11T20:13:07,407 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@70a14b1c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/hadoop.log.dir/,STOPPED}
2024-12-11T20:13:07,408 WARN  [BP-773969328-172.17.0.2-1733947940734 heartbeating to localhost/127.0.0.1:40619 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-11T20:13:07,408 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-11T20:13:07,408 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@20c72eb7 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-773969328-172.17.0.2-1733947940734:blk_1073741837_1013, datanode=DatanodeInfoWithStorage[127.0.0.1:34893,null,null])
java.io.InterruptedIOException: DestHost:destPort localhost:41499 , LocalHost:localPort 9fe0640122ec/172.17.0.2:0. Failed on local exception: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS)
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:936) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS)
	at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:963) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
Caused by: java.lang.InterruptedException: sleep interrupted
	at java.lang.Thread.sleep(Native Method) ~[?:?]
	at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-12-11T20:13:07,408 WARN  [BP-773969328-172.17.0.2-1733947940734 heartbeating to localhost/127.0.0.1:40619 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-773969328-172.17.0.2-1733947940734 (Datanode Uuid 669f8eed-2f96-4854-8c9a-69c5c7a3d0c2) service to localhost/127.0.0.1:40619
2024-12-11T20:13:07,408 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-11T20:13:07,409 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/cluster_f21598ea-49df-c0f8-b759-6a4f22597d28/data/data3/current/BP-773969328-172.17.0.2-1733947940734 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T20:13:07,409 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/cluster_f21598ea-49df-c0f8-b759-6a4f22597d28/data/data4/current/BP-773969328-172.17.0.2-1733947940734 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T20:13:07,409 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@20c72eb7 {}] datanode.BlockRecoveryWorker$1(605): recover Block: RecoveringBlock{BP-773969328-172.17.0.2-1733947940734:blk_1073741837_1013; getBlockSize()=85; corrupt=false; offset=-1; locs=[DatanodeInfoWithStorage[127.0.0.1:34893,null,null], DatanodeInfoWithStorage[127.0.0.1:37255,null,null]]; cachedLocs=[]} FAILED: java.io.IOException: No block pool offer service for bpid=BP-773969328-172.17.0.2-1733947940734
2024-12-11T20:13:07,410 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-11T20:13:07,410 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@20c72eb7 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-773969328-172.17.0.2-1733947940734:blk_1073741834_1010, datanode=DatanodeInfoWithStorage[127.0.0.1:34893,null,null])
java.io.IOException: No block pool offer service for bpid=BP-773969328-172.17.0.2-1733947940734
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.getDatanodeID(BlockRecoveryWorker.java:539) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$000(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:131) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:13:07,410 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@20c72eb7 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-773969328-172.17.0.2-1733947940734:blk_1073741834_1010, datanode=DatanodeInfoWithStorage[127.0.0.1:37255,null,null])
java.io.IOException: No block pool offer service for bpid=BP-773969328-172.17.0.2-1733947940734
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.getDatanodeID(BlockRecoveryWorker.java:539) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$000(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:131) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:13:07,410 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@20c72eb7 {}] datanode.BlockRecoveryWorker$1(605): recover Block: RecoveringBlock{BP-773969328-172.17.0.2-1733947940734:blk_1073741834_1010; getBlockSize()=85; corrupt=false; offset=-1; locs=[DatanodeInfoWithStorage[127.0.0.1:34893,null,null], DatanodeInfoWithStorage[127.0.0.1:37255,null,null]]; cachedLocs=[]} FAILED: java.io.IOException: All datanodes failed: block=BP-773969328-172.17.0.2-1733947940734:blk_1073741834_1010, datanodeids=[DatanodeInfoWithStorage[127.0.0.1:34893,null,null], DatanodeInfoWithStorage[127.0.0.1:37255,null,null]]
2024-12-11T20:13:07,412 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@a5ffcda{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T20:13:07,412 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@61d45840{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-11T20:13:07,412 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-11T20:13:07,413 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3fd17220{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-11T20:13:07,413 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7f55aa3b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/hadoop.log.dir/,STOPPED}
2024-12-11T20:13:07,415 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-11T20:13:07,415 WARN  [BP-773969328-172.17.0.2-1733947940734 heartbeating to localhost/127.0.0.1:40619 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-11T20:13:07,415 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-11T20:13:07,415 WARN  [BP-773969328-172.17.0.2-1733947940734 heartbeating to localhost/127.0.0.1:40619 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-773969328-172.17.0.2-1733947940734 (Datanode Uuid ce0d95cd-8c3d-4ad7-8b80-989a1cd7ec68) service to localhost/127.0.0.1:40619
2024-12-11T20:13:07,415 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/cluster_f21598ea-49df-c0f8-b759-6a4f22597d28/data/data9/current/BP-773969328-172.17.0.2-1733947940734 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T20:13:07,416 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/cluster_f21598ea-49df-c0f8-b759-6a4f22597d28/data/data10/current/BP-773969328-172.17.0.2-1733947940734 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T20:13:07,416 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-11T20:13:07,422 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@d607c51{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-11T20:13:07,423 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7ae0a20f{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-11T20:13:07,423 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-11T20:13:07,423 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2fd186ec{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-11T20:13:07,423 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@73b9709e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/hadoop.log.dir/,STOPPED}
2024-12-11T20:13:07,432 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-12-11T20:13:07,460 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-12-11T20:13:07,473 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=162 (was 81)
Potentially hanging thread: nioEventLoopGroup-18-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:40619 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-20-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-11
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-20-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:40619 from jenkins.hfs.3
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: HMaster-EventLoopGroup-7-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-21-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-5
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-10
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-19-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-7-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-5
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:44595
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-7-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:40619
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-20-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-21-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-19-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-5-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-5-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Close-WAL-Writer-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$899/0x00007f8084bf4210.run(Unknown Source)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-18-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-19-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:40619 from jenkins.hfs.2
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: HFileArchiver-6
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:40619 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.3@localhost:40619
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-7
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.1@localhost:44595
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.2@localhost:40619
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-18-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40619
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Timer for 'DataNode' metrics system
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: Close-WAL-Writer-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$899/0x00007f8084bf4210.run(Unknown Source)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-21-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-8
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-5-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40619
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40619
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40619
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-9
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=450 (was 402) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=207 (was 167) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=343 (was 940)
2024-12-11T20:13:07,482 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=162, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=207, ProcessCount=11, AvailableMemoryMB=343
2024-12-11T20:13:07,483 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-11T20:13:07,483 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/hadoop.log.dir so I do NOT create it in target/test-data/e6761f64-4d8d-e744-edc5-9c12798c8e6a
2024-12-11T20:13:07,483 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1ee7852b-0119-b3c4-d6ac-a05abe823bad/hadoop.tmp.dir so I do NOT create it in target/test-data/e6761f64-4d8d-e744-edc5-9c12798c8e6a
2024-12-11T20:13:07,483 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6761f64-4d8d-e744-edc5-9c12798c8e6a/cluster_e6efd374-2de3-88d0-f464-6076ba3d414f, deleteOnExit=true
2024-12-11T20:13:07,483 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-12-11T20:13:07,484 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6761f64-4d8d-e744-edc5-9c12798c8e6a/test.cache.data in system properties and HBase conf
2024-12-11T20:13:07,484 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6761f64-4d8d-e744-edc5-9c12798c8e6a/hadoop.tmp.dir in system properties and HBase conf
2024-12-11T20:13:07,484 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6761f64-4d8d-e744-edc5-9c12798c8e6a/hadoop.log.dir in system properties and HBase conf
2024-12-11T20:13:07,484 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6761f64-4d8d-e744-edc5-9c12798c8e6a/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-11T20:13:07,484 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6761f64-4d8d-e744-edc5-9c12798c8e6a/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-11T20:13:07,484 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-12-11T20:13:07,484 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-11T20:13:07,485 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6761f64-4d8d-e744-edc5-9c12798c8e6a/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-11T20:13:07,485 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6761f64-4d8d-e744-edc5-9c12798c8e6a/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-11T20:13:07,485 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6761f64-4d8d-e744-edc5-9c12798c8e6a/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-11T20:13:07,485 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6761f64-4d8d-e744-edc5-9c12798c8e6a/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-11T20:13:07,485 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6761f64-4d8d-e744-edc5-9c12798c8e6a/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-11T20:13:07,485 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6761f64-4d8d-e744-edc5-9c12798c8e6a/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-11T20:13:07,485 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6761f64-4d8d-e744-edc5-9c12798c8e6a/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-11T20:13:07,486 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6761f64-4d8d-e744-edc5-9c12798c8e6a/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-11T20:13:07,486 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6761f64-4d8d-e744-edc5-9c12798c8e6a/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-11T20:13:07,486 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6761f64-4d8d-e744-edc5-9c12798c8e6a/nfs.dump.dir in system properties and HBase conf
2024-12-11T20:13:07,486 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6761f64-4d8d-e744-edc5-9c12798c8e6a/java.io.tmpdir in system properties and HBase conf
2024-12-11T20:13:07,486 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6761f64-4d8d-e744-edc5-9c12798c8e6a/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-11T20:13:07,486 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6761f64-4d8d-e744-edc5-9c12798c8e6a/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-11T20:13:07,486 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6761f64-4d8d-e744-edc5-9c12798c8e6a/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-11T20:13:07,500 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-11T20:13:07,576 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T20:13:07,581 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-11T20:13:07,582 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-11T20:13:07,582 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-11T20:13:07,582 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-11T20:13:07,583 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T20:13:07,583 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5e4fac25{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6761f64-4d8d-e744-edc5-9c12798c8e6a/hadoop.log.dir/,AVAILABLE}
2024-12-11T20:13:07,583 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1a59d25d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-11T20:13:07,699 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@533cae40{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6761f64-4d8d-e744-edc5-9c12798c8e6a/java.io.tmpdir/jetty-localhost-46137-hadoop-hdfs-3_4_1-tests_jar-_-any-7111879557827679064/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-11T20:13:07,699 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1e98c3cf{HTTP/1.1, (http/1.1)}{localhost:46137}
2024-12-11T20:13:07,699 INFO  [Time-limited test {}] server.Server(415): Started @151295ms
2024-12-11T20:13:07,718 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-11T20:13:07,820 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T20:13:07,824 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-11T20:13:07,825 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-11T20:13:07,825 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-11T20:13:07,826 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-11T20:13:07,826 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@54562211{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6761f64-4d8d-e744-edc5-9c12798c8e6a/hadoop.log.dir/,AVAILABLE}
2024-12-11T20:13:07,827 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@32d7b29f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-11T20:13:07,947 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@624b3986{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6761f64-4d8d-e744-edc5-9c12798c8e6a/java.io.tmpdir/jetty-localhost-34041-hadoop-hdfs-3_4_1-tests_jar-_-any-1105413195135617452/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T20:13:07,947 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7e13d66c{HTTP/1.1, (http/1.1)}{localhost:34041}
2024-12-11T20:13:07,948 INFO  [Time-limited test {}] server.Server(415): Started @151543ms
2024-12-11T20:13:07,949 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-11T20:13:07,981 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T20:13:07,985 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-11T20:13:07,988 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-11T20:13:07,988 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-11T20:13:07,988 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-11T20:13:07,989 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@33392a77{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6761f64-4d8d-e744-edc5-9c12798c8e6a/hadoop.log.dir/,AVAILABLE}
2024-12-11T20:13:07,989 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@44bd8b73{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-11T20:13:08,029 WARN  [Thread-1193 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6761f64-4d8d-e744-edc5-9c12798c8e6a/cluster_e6efd374-2de3-88d0-f464-6076ba3d414f/data/data1/current/BP-558819663-172.17.0.2-1733947987520/current, will proceed with Du for space computation calculation, 
2024-12-11T20:13:08,030 WARN  [Thread-1194 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6761f64-4d8d-e744-edc5-9c12798c8e6a/cluster_e6efd374-2de3-88d0-f464-6076ba3d414f/data/data2/current/BP-558819663-172.17.0.2-1733947987520/current, will proceed with Du for space computation calculation, 
2024-12-11T20:13:08,047 WARN  [Thread-1172 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-11T20:13:08,049 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xadb456ac20b5d65b with lease ID 0xa43fd1a80235cb38: Processing first storage report for DS-7c862880-c949-4d2b-b189-775d30f1b8af from datanode DatanodeRegistration(127.0.0.1:33075, datanodeUuid=4c3e95e7-fe9c-4d12-ba2c-9bb9508f688f, infoPort=42747, infoSecurePort=0, ipcPort=40153, storageInfo=lv=-57;cid=testClusterID;nsid=1183363567;c=1733947987520)
2024-12-11T20:13:08,049 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xadb456ac20b5d65b with lease ID 0xa43fd1a80235cb38: from storage DS-7c862880-c949-4d2b-b189-775d30f1b8af node DatanodeRegistration(127.0.0.1:33075, datanodeUuid=4c3e95e7-fe9c-4d12-ba2c-9bb9508f688f, infoPort=42747, infoSecurePort=0, ipcPort=40153, storageInfo=lv=-57;cid=testClusterID;nsid=1183363567;c=1733947987520), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T20:13:08,049 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xadb456ac20b5d65b with lease ID 0xa43fd1a80235cb38: Processing first storage report for DS-3ff62e98-e67b-4e4c-a8eb-73b19b99f495 from datanode DatanodeRegistration(127.0.0.1:33075, datanodeUuid=4c3e95e7-fe9c-4d12-ba2c-9bb9508f688f, infoPort=42747, infoSecurePort=0, ipcPort=40153, storageInfo=lv=-57;cid=testClusterID;nsid=1183363567;c=1733947987520)
2024-12-11T20:13:08,049 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xadb456ac20b5d65b with lease ID 0xa43fd1a80235cb38: from storage DS-3ff62e98-e67b-4e4c-a8eb-73b19b99f495 node DatanodeRegistration(127.0.0.1:33075, datanodeUuid=4c3e95e7-fe9c-4d12-ba2c-9bb9508f688f, infoPort=42747, infoSecurePort=0, ipcPort=40153, storageInfo=lv=-57;cid=testClusterID;nsid=1183363567;c=1733947987520), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T20:13:08,110 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6d74671d{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6761f64-4d8d-e744-edc5-9c12798c8e6a/java.io.tmpdir/jetty-localhost-34321-hadoop-hdfs-3_4_1-tests_jar-_-any-1097953830915503908/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T20:13:08,111 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4247f9cf{HTTP/1.1, (http/1.1)}{localhost:34321}
2024-12-11T20:13:08,111 INFO  [Time-limited test {}] server.Server(415): Started @151706ms
2024-12-11T20:13:08,112 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-11T20:13:08,194 WARN  [Thread-1219 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6761f64-4d8d-e744-edc5-9c12798c8e6a/cluster_e6efd374-2de3-88d0-f464-6076ba3d414f/data/data3/current/BP-558819663-172.17.0.2-1733947987520/current, will proceed with Du for space computation calculation, 
2024-12-11T20:13:08,194 WARN  [Thread-1220 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6761f64-4d8d-e744-edc5-9c12798c8e6a/cluster_e6efd374-2de3-88d0-f464-6076ba3d414f/data/data4/current/BP-558819663-172.17.0.2-1733947987520/current, will proceed with Du for space computation calculation, 
2024-12-11T20:13:08,210 WARN  [Thread-1208 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-11T20:13:08,213 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbf7b83f77877b6bd with lease ID 0xa43fd1a80235cb39: Processing first storage report for DS-5af1d7d7-8339-47f7-888c-d7823d3eef19 from datanode DatanodeRegistration(127.0.0.1:37927, datanodeUuid=c24dc4a7-9b08-451a-82ef-036733e0d594, infoPort=42137, infoSecurePort=0, ipcPort=37447, storageInfo=lv=-57;cid=testClusterID;nsid=1183363567;c=1733947987520)
2024-12-11T20:13:08,213 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbf7b83f77877b6bd with lease ID 0xa43fd1a80235cb39: from storage DS-5af1d7d7-8339-47f7-888c-d7823d3eef19 node DatanodeRegistration(127.0.0.1:37927, datanodeUuid=c24dc4a7-9b08-451a-82ef-036733e0d594, infoPort=42137, infoSecurePort=0, ipcPort=37447, storageInfo=lv=-57;cid=testClusterID;nsid=1183363567;c=1733947987520), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T20:13:08,213 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbf7b83f77877b6bd with lease ID 0xa43fd1a80235cb39: Processing first storage report for DS-c33d6bb7-cece-417c-82dd-c95497ef7faf from datanode DatanodeRegistration(127.0.0.1:37927, datanodeUuid=c24dc4a7-9b08-451a-82ef-036733e0d594, infoPort=42137, infoSecurePort=0, ipcPort=37447, storageInfo=lv=-57;cid=testClusterID;nsid=1183363567;c=1733947987520)
2024-12-11T20:13:08,213 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbf7b83f77877b6bd with lease ID 0xa43fd1a80235cb39: from storage DS-c33d6bb7-cece-417c-82dd-c95497ef7faf node DatanodeRegistration(127.0.0.1:37927, datanodeUuid=c24dc4a7-9b08-451a-82ef-036733e0d594, infoPort=42137, infoSecurePort=0, ipcPort=37447, storageInfo=lv=-57;cid=testClusterID;nsid=1183363567;c=1733947987520), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T20:13:08,237 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6761f64-4d8d-e744-edc5-9c12798c8e6a
2024-12-11T20:13:08,240 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6761f64-4d8d-e744-edc5-9c12798c8e6a/cluster_e6efd374-2de3-88d0-f464-6076ba3d414f/zookeeper_0, clientPort=58530, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6761f64-4d8d-e744-edc5-9c12798c8e6a/cluster_e6efd374-2de3-88d0-f464-6076ba3d414f/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6761f64-4d8d-e744-edc5-9c12798c8e6a/cluster_e6efd374-2de3-88d0-f464-6076ba3d414f/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-11T20:13:08,241 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=58530
2024-12-11T20:13:08,241 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T20:13:08,243 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T20:13:08,251 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33075 is added to blk_1073741825_1001 (size=7)
2024-12-11T20:13:08,252 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37927 is added to blk_1073741825_1001 (size=7)
2024-12-11T20:13:08,253 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3 with version=8
2024-12-11T20:13:08,253 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/hbase-staging
2024-12-11T20:13:08,255 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/9fe0640122ec:0 server-side Connection retries=45
2024-12-11T20:13:08,255 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-11T20:13:08,255 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-11T20:13:08,255 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-11T20:13:08,255 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-11T20:13:08,255 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-11T20:13:08,255 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-12-11T20:13:08,256 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-11T20:13:08,256 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40715
2024-12-11T20:13:08,258 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:40715 connecting to ZooKeeper ensemble=127.0.0.1:58530
2024-12-11T20:13:08,264 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:407150x0, quorum=127.0.0.1:58530, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-11T20:13:08,264 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:40715-0x100caf8ab8e0000 connected
2024-12-11T20:13:08,265 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:08,271 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:08,277 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T20:13:08,279 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T20:13:08,281 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40715-0x100caf8ab8e0000, quorum=127.0.0.1:58530, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-11T20:13:08,281 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3, hbase.cluster.distributed=false
2024-12-11T20:13:08,284 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40715-0x100caf8ab8e0000, quorum=127.0.0.1:58530, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-11T20:13:08,284 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40715
2024-12-11T20:13:08,285 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40715
2024-12-11T20:13:08,285 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40715
2024-12-11T20:13:08,286 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40715
2024-12-11T20:13:08,286 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40715
2024-12-11T20:13:08,303 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/9fe0640122ec:0 server-side Connection retries=45
2024-12-11T20:13:08,304 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-11T20:13:08,304 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-11T20:13:08,304 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-11T20:13:08,304 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-11T20:13:08,304 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-11T20:13:08,304 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-11T20:13:08,304 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-11T20:13:08,305 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45215
2024-12-11T20:13:08,306 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45215 connecting to ZooKeeper ensemble=127.0.0.1:58530
2024-12-11T20:13:08,307 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T20:13:08,310 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T20:13:08,316 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:452150x0, quorum=127.0.0.1:58530, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-11T20:13:08,316 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:452150x0, quorum=127.0.0.1:58530, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-11T20:13:08,317 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45215-0x100caf8ab8e0001 connected
2024-12-11T20:13:08,317 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-11T20:13:08,320 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-11T20:13:08,321 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45215-0x100caf8ab8e0001, quorum=127.0.0.1:58530, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-11T20:13:08,322 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45215-0x100caf8ab8e0001, quorum=127.0.0.1:58530, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-11T20:13:08,323 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45215
2024-12-11T20:13:08,324 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45215
2024-12-11T20:13:08,326 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45215
2024-12-11T20:13:08,327 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45215
2024-12-11T20:13:08,327 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45215
2024-12-11T20:13:08,342 DEBUG [M:0;9fe0640122ec:40715 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;9fe0640122ec:40715
2024-12-11T20:13:08,342 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/9fe0640122ec,40715,1733947988255
2024-12-11T20:13:08,344 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40715-0x100caf8ab8e0000, quorum=127.0.0.1:58530, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-11T20:13:08,344 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45215-0x100caf8ab8e0001, quorum=127.0.0.1:58530, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-11T20:13:08,345 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40715-0x100caf8ab8e0000, quorum=127.0.0.1:58530, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/9fe0640122ec,40715,1733947988255
2024-12-11T20:13:08,347 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45215-0x100caf8ab8e0001, quorum=127.0.0.1:58530, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-11T20:13:08,347 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40715-0x100caf8ab8e0000, quorum=127.0.0.1:58530, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:13:08,347 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45215-0x100caf8ab8e0001, quorum=127.0.0.1:58530, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:13:08,347 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40715-0x100caf8ab8e0000, quorum=127.0.0.1:58530, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-11T20:13:08,348 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/9fe0640122ec,40715,1733947988255 from backup master directory
2024-12-11T20:13:08,349 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40715-0x100caf8ab8e0000, quorum=127.0.0.1:58530, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/9fe0640122ec,40715,1733947988255
2024-12-11T20:13:08,349 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45215-0x100caf8ab8e0001, quorum=127.0.0.1:58530, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-11T20:13:08,349 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40715-0x100caf8ab8e0000, quorum=127.0.0.1:58530, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-11T20:13:08,349 WARN  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-11T20:13:08,349 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=9fe0640122ec,40715,1733947988255
2024-12-11T20:13:08,356 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/hbase.id] with ID: 06e81d67-f6fa-4ec8-babb-ba6eb56ef592
2024-12-11T20:13:08,356 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/.tmp/hbase.id
2024-12-11T20:13:08,366 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37927 is added to blk_1073741826_1002 (size=42)
2024-12-11T20:13:08,367 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33075 is added to blk_1073741826_1002 (size=42)
2024-12-11T20:13:08,367 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/.tmp/hbase.id]:[hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/hbase.id]
2024-12-11T20:13:08,385 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T20:13:08,385 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-12-11T20:13:08,387 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms.
2024-12-11T20:13:08,391 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40715-0x100caf8ab8e0000, quorum=127.0.0.1:58530, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:13:08,391 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45215-0x100caf8ab8e0001, quorum=127.0.0.1:58530, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:13:08,407 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37927 is added to blk_1073741827_1003 (size=196)
2024-12-11T20:13:08,408 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33075 is added to blk_1073741827_1003 (size=196)
2024-12-11T20:13:08,809 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-11T20:13:08,810 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-11T20:13:08,810 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-11T20:13:08,826 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37927 is added to blk_1073741828_1004 (size=1189)
2024-12-11T20:13:08,829 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33075 is added to blk_1073741828_1004 (size=1189)
2024-12-11T20:13:08,830 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/MasterData/data/master/store
2024-12-11T20:13:08,838 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37927 is added to blk_1073741829_1005 (size=34)
2024-12-11T20:13:08,839 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33075 is added to blk_1073741829_1005 (size=34)
2024-12-11T20:13:08,839 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T20:13:08,839 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-11T20:13:08,839 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T20:13:08,839 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T20:13:08,840 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-11T20:13:08,840 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T20:13:08,840 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T20:13:08,840 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733947988839Disabling compacts and flushes for region at 1733947988839Disabling writes for close at 1733947988840 (+1 ms)Writing region close event to WAL at 1733947988840Closed at 1733947988840
2024-12-11T20:13:08,841 WARN  [master/9fe0640122ec:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/MasterData/data/master/store/.initializing
2024-12-11T20:13:08,841 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/MasterData/WALs/9fe0640122ec,40715,1733947988255
2024-12-11T20:13:08,844 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9fe0640122ec%2C40715%2C1733947988255, suffix=, logDir=hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/MasterData/WALs/9fe0640122ec,40715,1733947988255, archiveDir=hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/MasterData/oldWALs, maxLogs=10
2024-12-11T20:13:08,844 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9fe0640122ec%2C40715%2C1733947988255.1733947988844
2024-12-11T20:13:08,851 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/MasterData/WALs/9fe0640122ec,40715,1733947988255/9fe0640122ec%2C40715%2C1733947988255.1733947988844
2024-12-11T20:13:08,853 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42747:42747),(127.0.0.1/127.0.0.1:42137:42137)]
2024-12-11T20:13:08,854 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-11T20:13:08,854 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T20:13:08,854 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T20:13:08,854 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T20:13:08,856 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-11T20:13:08,857 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-11T20:13:08,857 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:13:08,858 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T20:13:08,858 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-11T20:13:08,859 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-11T20:13:08,859 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:13:08,859 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-11T20:13:08,859 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-11T20:13:08,860 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-11T20:13:08,860 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:13:08,861 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-11T20:13:08,861 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-11T20:13:08,862 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-11T20:13:08,862 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:13:08,863 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-11T20:13:08,863 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T20:13:08,863 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-11T20:13:08,864 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-11T20:13:08,865 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T20:13:08,865 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T20:13:08,865 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-11T20:13:08,866 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T20:13:08,868 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-11T20:13:08,869 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=820443, jitterRate=0.043247222900390625}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-11T20:13:08,869 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1733947988854Initializing all the Stores at 1733947988855 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733947988855Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733947988855Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733947988855Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733947988855Cleaning up temporary data from old regions at 1733947988865 (+10 ms)Region opened successfully at 1733947988869 (+4 ms)
2024-12-11T20:13:08,870 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-11T20:13:08,872 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f1b5d0e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9fe0640122ec/172.17.0.2:0
2024-12-11T20:13:08,873 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-12-11T20:13:08,873 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-11T20:13:08,873 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-11T20:13:08,874 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-11T20:13:08,874 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-12-11T20:13:08,874 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-12-11T20:13:08,874 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-11T20:13:08,876 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-11T20:13:08,877 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40715-0x100caf8ab8e0000, quorum=127.0.0.1:58530, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-11T20:13:08,878 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-12-11T20:13:08,878 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-11T20:13:08,879 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40715-0x100caf8ab8e0000, quorum=127.0.0.1:58530, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-11T20:13:08,880 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-12-11T20:13:08,880 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-11T20:13:08,881 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40715-0x100caf8ab8e0000, quorum=127.0.0.1:58530, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-11T20:13:08,882 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-12-11T20:13:08,884 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40715-0x100caf8ab8e0000, quorum=127.0.0.1:58530, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-11T20:13:08,885 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-12-11T20:13:08,886 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40715-0x100caf8ab8e0000, quorum=127.0.0.1:58530, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-11T20:13:08,887 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-11T20:13:08,889 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40715-0x100caf8ab8e0000, quorum=127.0.0.1:58530, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-11T20:13:08,889 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40715-0x100caf8ab8e0000, quorum=127.0.0.1:58530, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:13:08,889 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45215-0x100caf8ab8e0001, quorum=127.0.0.1:58530, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-11T20:13:08,889 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45215-0x100caf8ab8e0001, quorum=127.0.0.1:58530, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:13:08,889 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=9fe0640122ec,40715,1733947988255, sessionid=0x100caf8ab8e0000, setting cluster-up flag (Was=false)
2024-12-11T20:13:08,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40715-0x100caf8ab8e0000, quorum=127.0.0.1:58530, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:13:08,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45215-0x100caf8ab8e0001, quorum=127.0.0.1:58530, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:13:08,897 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-11T20:13:08,898 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9fe0640122ec,40715,1733947988255
2024-12-11T20:13:08,901 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40715-0x100caf8ab8e0000, quorum=127.0.0.1:58530, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:13:08,901 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45215-0x100caf8ab8e0001, quorum=127.0.0.1:58530, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:13:08,905 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-11T20:13:08,906 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9fe0640122ec,40715,1733947988255
2024-12-11T20:13:08,908 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-12-11T20:13:08,909 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-12-11T20:13:08,910 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-12-11T20:13:08,910 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-11T20:13:08,910 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 9fe0640122ec,40715,1733947988255
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-11T20:13:08,911 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/9fe0640122ec:0, corePoolSize=5, maxPoolSize=5
2024-12-11T20:13:08,911 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/9fe0640122ec:0, corePoolSize=5, maxPoolSize=5
2024-12-11T20:13:08,911 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/9fe0640122ec:0, corePoolSize=5, maxPoolSize=5
2024-12-11T20:13:08,911 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/9fe0640122ec:0, corePoolSize=5, maxPoolSize=5
2024-12-11T20:13:08,911 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/9fe0640122ec:0, corePoolSize=10, maxPoolSize=10
2024-12-11T20:13:08,911 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:13:08,911 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/9fe0640122ec:0, corePoolSize=2, maxPoolSize=2
2024-12-11T20:13:08,911 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:13:08,912 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733948018912
2024-12-11T20:13:08,912 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-11T20:13:08,912 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-11T20:13:08,913 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-11T20:13:08,913 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-11T20:13:08,913 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-11T20:13:08,913 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-11T20:13:08,913 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-11T20:13:08,913 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-11T20:13:08,913 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-12-11T20:13:08,914 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-11T20:13:08,914 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-11T20:13:08,914 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-11T20:13:08,914 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-11T20:13:08,914 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-11T20:13:08,914 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:13:08,914 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/9fe0640122ec:0:becomeActiveMaster-HFileCleaner.large.0-1733947988914,5,FailOnTimeoutGroup]
2024-12-11T20:13:08,915 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/9fe0640122ec:0:becomeActiveMaster-HFileCleaner.small.0-1733947988914,5,FailOnTimeoutGroup]
2024-12-11T20:13:08,915 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-11T20:13:08,914 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-11T20:13:08,915 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-11T20:13:08,915 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-11T20:13:08,915 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-11T20:13:08,920 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33075 is added to blk_1073741831_1007 (size=1321)
2024-12-11T20:13:08,921 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37927 is added to blk_1073741831_1007 (size=1321)
2024-12-11T20:13:08,921 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-12-11T20:13:08,922 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3
2024-12-11T20:13:08,927 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33075 is added to blk_1073741832_1008 (size=32)
2024-12-11T20:13:08,927 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37927 is added to blk_1073741832_1008 (size=32)
2024-12-11T20:13:08,928 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T20:13:08,929 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-11T20:13:08,930 INFO  [RS:0;9fe0640122ec:45215 {}] regionserver.HRegionServer(746): ClusterId : 06e81d67-f6fa-4ec8-babb-ba6eb56ef592
2024-12-11T20:13:08,930 DEBUG [RS:0;9fe0640122ec:45215 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-11T20:13:08,930 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-11T20:13:08,930 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:13:08,931 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T20:13:08,931 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-11T20:13:08,932 DEBUG [RS:0;9fe0640122ec:45215 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-11T20:13:08,932 DEBUG [RS:0;9fe0640122ec:45215 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-11T20:13:08,932 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-11T20:13:08,932 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:13:08,933 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T20:13:08,933 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-11T20:13:08,934 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-11T20:13:08,934 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:13:08,934 DEBUG [RS:0;9fe0640122ec:45215 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-11T20:13:08,934 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T20:13:08,934 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-11T20:13:08,934 DEBUG [RS:0;9fe0640122ec:45215 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f183177, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9fe0640122ec/172.17.0.2:0
2024-12-11T20:13:08,935 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-11T20:13:08,935 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:13:08,936 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T20:13:08,936 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-11T20:13:08,937 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/data/hbase/meta/1588230740
2024-12-11T20:13:08,937 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/data/hbase/meta/1588230740
2024-12-11T20:13:08,938 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-11T20:13:08,938 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-11T20:13:08,939 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-11T20:13:08,940 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-11T20:13:08,942 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-11T20:13:08,942 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=776659, jitterRate=-0.01242758333683014}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-11T20:13:08,943 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1733947988928Initializing all the Stores at 1733947988928Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733947988928Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733947988929 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733947988929Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733947988929Cleaning up temporary data from old regions at 1733947988938 (+9 ms)Region opened successfully at 1733947988943 (+5 ms)
2024-12-11T20:13:08,943 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-11T20:13:08,943 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-11T20:13:08,943 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-11T20:13:08,943 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-11T20:13:08,943 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-11T20:13:08,944 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-11T20:13:08,944 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733947988943Disabling compacts and flushes for region at 1733947988943Disabling writes for close at 1733947988943Writing region close event to WAL at 1733947988944 (+1 ms)Closed at 1733947988944
2024-12-11T20:13:08,945 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-11T20:13:08,945 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-12-11T20:13:08,945 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-11T20:13:08,946 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-11T20:13:08,948 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-11T20:13:08,948 DEBUG [RS:0;9fe0640122ec:45215 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;9fe0640122ec:45215
2024-12-11T20:13:08,948 INFO  [RS:0;9fe0640122ec:45215 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-11T20:13:08,948 INFO  [RS:0;9fe0640122ec:45215 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-11T20:13:08,948 DEBUG [RS:0;9fe0640122ec:45215 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-11T20:13:08,949 INFO  [RS:0;9fe0640122ec:45215 {}] regionserver.HRegionServer(2659): reportForDuty to master=9fe0640122ec,40715,1733947988255 with port=45215, startcode=1733947988303
2024-12-11T20:13:08,949 DEBUG [RS:0;9fe0640122ec:45215 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-11T20:13:08,951 INFO  [HMaster-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39833, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.4 (auth:SIMPLE), service=RegionServerStatusService
2024-12-11T20:13:08,952 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40715 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9fe0640122ec,45215,1733947988303
2024-12-11T20:13:08,952 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40715 {}] master.ServerManager(517): Registering regionserver=9fe0640122ec,45215,1733947988303
2024-12-11T20:13:08,954 DEBUG [RS:0;9fe0640122ec:45215 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3
2024-12-11T20:13:08,954 DEBUG [RS:0;9fe0640122ec:45215 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:42243
2024-12-11T20:13:08,954 DEBUG [RS:0;9fe0640122ec:45215 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-11T20:13:08,955 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40715-0x100caf8ab8e0000, quorum=127.0.0.1:58530, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-11T20:13:08,956 DEBUG [RS:0;9fe0640122ec:45215 {}] zookeeper.ZKUtil(111): regionserver:45215-0x100caf8ab8e0001, quorum=127.0.0.1:58530, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9fe0640122ec,45215,1733947988303
2024-12-11T20:13:08,956 WARN  [RS:0;9fe0640122ec:45215 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-11T20:13:08,956 INFO  [RS:0;9fe0640122ec:45215 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-11T20:13:08,956 DEBUG [RS:0;9fe0640122ec:45215 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303
2024-12-11T20:13:08,956 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9fe0640122ec,45215,1733947988303]
2024-12-11T20:13:08,959 INFO  [RS:0;9fe0640122ec:45215 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-11T20:13:08,961 INFO  [RS:0;9fe0640122ec:45215 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-11T20:13:08,961 INFO  [RS:0;9fe0640122ec:45215 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-11T20:13:08,961 INFO  [RS:0;9fe0640122ec:45215 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-11T20:13:08,961 INFO  [RS:0;9fe0640122ec:45215 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-11T20:13:08,962 INFO  [RS:0;9fe0640122ec:45215 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-11T20:13:08,962 INFO  [RS:0;9fe0640122ec:45215 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-11T20:13:08,962 DEBUG [RS:0;9fe0640122ec:45215 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:13:08,962 DEBUG [RS:0;9fe0640122ec:45215 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:13:08,962 DEBUG [RS:0;9fe0640122ec:45215 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:13:08,962 DEBUG [RS:0;9fe0640122ec:45215 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:13:08,963 DEBUG [RS:0;9fe0640122ec:45215 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:13:08,963 DEBUG [RS:0;9fe0640122ec:45215 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9fe0640122ec:0, corePoolSize=2, maxPoolSize=2
2024-12-11T20:13:08,963 DEBUG [RS:0;9fe0640122ec:45215 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:13:08,963 DEBUG [RS:0;9fe0640122ec:45215 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:13:08,963 DEBUG [RS:0;9fe0640122ec:45215 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:13:08,963 DEBUG [RS:0;9fe0640122ec:45215 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:13:08,963 DEBUG [RS:0;9fe0640122ec:45215 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:13:08,963 DEBUG [RS:0;9fe0640122ec:45215 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:13:08,963 DEBUG [RS:0;9fe0640122ec:45215 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9fe0640122ec:0, corePoolSize=3, maxPoolSize=3
2024-12-11T20:13:08,963 DEBUG [RS:0;9fe0640122ec:45215 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9fe0640122ec:0, corePoolSize=3, maxPoolSize=3
2024-12-11T20:13:08,964 INFO  [RS:0;9fe0640122ec:45215 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-11T20:13:08,964 INFO  [RS:0;9fe0640122ec:45215 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-11T20:13:08,965 INFO  [RS:0;9fe0640122ec:45215 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-11T20:13:08,965 INFO  [RS:0;9fe0640122ec:45215 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-11T20:13:08,965 INFO  [RS:0;9fe0640122ec:45215 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-11T20:13:08,965 INFO  [RS:0;9fe0640122ec:45215 {}] hbase.ChoreService(168): Chore ScheduledChore name=9fe0640122ec,45215,1733947988303-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-11T20:13:08,988 INFO  [RS:0;9fe0640122ec:45215 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-11T20:13:08,988 INFO  [RS:0;9fe0640122ec:45215 {}] hbase.ChoreService(168): Chore ScheduledChore name=9fe0640122ec,45215,1733947988303-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-11T20:13:08,988 INFO  [RS:0;9fe0640122ec:45215 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T20:13:08,988 INFO  [RS:0;9fe0640122ec:45215 {}] regionserver.Replication(171): 9fe0640122ec,45215,1733947988303 started
2024-12-11T20:13:09,004 INFO  [RS:0;9fe0640122ec:45215 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T20:13:09,005 INFO  [RS:0;9fe0640122ec:45215 {}] regionserver.HRegionServer(1482): Serving as 9fe0640122ec,45215,1733947988303, RpcServer on 9fe0640122ec/172.17.0.2:45215, sessionid=0x100caf8ab8e0001
2024-12-11T20:13:09,005 DEBUG [RS:0;9fe0640122ec:45215 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-11T20:13:09,005 DEBUG [RS:0;9fe0640122ec:45215 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9fe0640122ec,45215,1733947988303
2024-12-11T20:13:09,005 DEBUG [RS:0;9fe0640122ec:45215 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9fe0640122ec,45215,1733947988303'
2024-12-11T20:13:09,005 DEBUG [RS:0;9fe0640122ec:45215 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-11T20:13:09,005 DEBUG [RS:0;9fe0640122ec:45215 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-11T20:13:09,006 DEBUG [RS:0;9fe0640122ec:45215 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-11T20:13:09,006 DEBUG [RS:0;9fe0640122ec:45215 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-11T20:13:09,006 DEBUG [RS:0;9fe0640122ec:45215 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9fe0640122ec,45215,1733947988303
2024-12-11T20:13:09,006 DEBUG [RS:0;9fe0640122ec:45215 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9fe0640122ec,45215,1733947988303'
2024-12-11T20:13:09,006 DEBUG [RS:0;9fe0640122ec:45215 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-11T20:13:09,006 DEBUG [RS:0;9fe0640122ec:45215 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-11T20:13:09,006 DEBUG [RS:0;9fe0640122ec:45215 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-11T20:13:09,006 INFO  [RS:0;9fe0640122ec:45215 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-11T20:13:09,006 INFO  [RS:0;9fe0640122ec:45215 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-11T20:13:09,098 WARN  [9fe0640122ec:40715 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-12-11T20:13:09,109 INFO  [RS:0;9fe0640122ec:45215 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9fe0640122ec%2C45215%2C1733947988303, suffix=, logDir=hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303, archiveDir=hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/oldWALs, maxLogs=32
2024-12-11T20:13:09,109 INFO  [RS:0;9fe0640122ec:45215 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9fe0640122ec%2C45215%2C1733947988303.1733947989109
2024-12-11T20:13:09,115 INFO  [RS:0;9fe0640122ec:45215 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.1733947989109
2024-12-11T20:13:09,116 DEBUG [RS:0;9fe0640122ec:45215 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42747:42747),(127.0.0.1/127.0.0.1:42137:42137)]
2024-12-11T20:13:09,266 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:09,271 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:09,348 DEBUG [9fe0640122ec:40715 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-11T20:13:09,349 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=9fe0640122ec,45215,1733947988303
2024-12-11T20:13:09,350 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9fe0640122ec,45215,1733947988303, state=OPENING
2024-12-11T20:13:09,352 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-11T20:13:09,354 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40715-0x100caf8ab8e0000, quorum=127.0.0.1:58530, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:13:09,354 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45215-0x100caf8ab8e0001, quorum=127.0.0.1:58530, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:13:09,354 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-11T20:13:09,354 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-11T20:13:09,354 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-11T20:13:09,354 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=9fe0640122ec,45215,1733947988303}]
2024-12-11T20:13:09,508 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-11T20:13:09,510 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59597, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-11T20:13:09,514 INFO  [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-12-11T20:13:09,514 INFO  [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-11T20:13:09,516 INFO  [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9fe0640122ec%2C45215%2C1733947988303.meta, suffix=.meta, logDir=hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303, archiveDir=hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/oldWALs, maxLogs=32
2024-12-11T20:13:09,516 INFO  [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 9fe0640122ec%2C45215%2C1733947988303.meta.1733947989516.meta
2024-12-11T20:13:09,522 INFO  [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.meta.1733947989516.meta
2024-12-11T20:13:09,523 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42137:42137),(127.0.0.1/127.0.0.1:42747:42747)]
2024-12-11T20:13:09,524 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-11T20:13:09,525 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-11T20:13:09,525 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-11T20:13:09,525 INFO  [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-11T20:13:09,525 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-11T20:13:09,525 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T20:13:09,525 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-12-11T20:13:09,525 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-12-11T20:13:09,527 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-11T20:13:09,528 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-11T20:13:09,528 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:13:09,528 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T20:13:09,529 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-11T20:13:09,529 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-11T20:13:09,529 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:13:09,530 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T20:13:09,530 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-11T20:13:09,531 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-11T20:13:09,531 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:13:09,531 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T20:13:09,532 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-11T20:13:09,532 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-11T20:13:09,532 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:13:09,533 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T20:13:09,533 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-11T20:13:09,534 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/data/hbase/meta/1588230740
2024-12-11T20:13:09,535 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/data/hbase/meta/1588230740
2024-12-11T20:13:09,537 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-11T20:13:09,537 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-11T20:13:09,537 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-11T20:13:09,539 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-11T20:13:09,540 INFO  [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=869584, jitterRate=0.10573382675647736}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-11T20:13:09,540 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-12-11T20:13:09,541 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1733947989526Writing region info on filesystem at 1733947989526Initializing all the Stores at 1733947989526Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733947989526Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733947989527 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733947989527Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733947989527Cleaning up temporary data from old regions at 1733947989537 (+10 ms)Running coprocessor post-open hooks at 1733947989540 (+3 ms)Region opened successfully at 1733947989541 (+1 ms)
2024-12-11T20:13:09,543 INFO  [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733947989508
2024-12-11T20:13:09,546 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-11T20:13:09,546 INFO  [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-12-11T20:13:09,547 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=9fe0640122ec,45215,1733947988303
2024-12-11T20:13:09,548 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9fe0640122ec,45215,1733947988303, state=OPEN
2024-12-11T20:13:09,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45215-0x100caf8ab8e0001, quorum=127.0.0.1:58530, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-11T20:13:09,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40715-0x100caf8ab8e0000, quorum=127.0.0.1:58530, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-11T20:13:09,555 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-11T20:13:09,555 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-11T20:13:09,555 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=9fe0640122ec,45215,1733947988303
2024-12-11T20:13:09,559 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-12-11T20:13:09,559 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9fe0640122ec,45215,1733947988303 in 201 msec
2024-12-11T20:13:09,561 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-12-11T20:13:09,561 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 614 msec
2024-12-11T20:13:09,562 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-11T20:13:09,562 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-12-11T20:13:09,564 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-11T20:13:09,564 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9fe0640122ec,45215,1733947988303, seqNum=-1]
2024-12-11T20:13:09,564 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-11T20:13:09,565 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-11-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47649, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-11T20:13:09,571 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 661 msec
2024-12-11T20:13:09,571 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733947989571, completionTime=-1
2024-12-11T20:13:09,571 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-11T20:13:09,571 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-12-11T20:13:09,573 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-12-11T20:13:09,573 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733948049573
2024-12-11T20:13:09,573 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733948109573
2024-12-11T20:13:09,573 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec
2024-12-11T20:13:09,574 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9fe0640122ec,40715,1733947988255-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-11T20:13:09,574 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9fe0640122ec,40715,1733947988255-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T20:13:09,574 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9fe0640122ec,40715,1733947988255-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T20:13:09,574 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-9fe0640122ec:40715, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T20:13:09,574 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-11T20:13:09,574 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-12-11T20:13:09,576 DEBUG [master/9fe0640122ec:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-12-11T20:13:09,578 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.229sec
2024-12-11T20:13:09,578 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-11T20:13:09,578 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-11T20:13:09,578 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-11T20:13:09,578 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-11T20:13:09,578 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-11T20:13:09,578 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9fe0640122ec,40715,1733947988255-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-11T20:13:09,578 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9fe0640122ec,40715,1733947988255-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-11T20:13:09,580 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-12-11T20:13:09,580 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-11T20:13:09,580 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9fe0640122ec,40715,1733947988255-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T20:13:09,630 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4136ef12, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-11T20:13:09,630 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9fe0640122ec,40715,-1 for getting cluster id
2024-12-11T20:13:09,631 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-12-11T20:13:09,635 DEBUG [HMaster-EventLoopGroup-10-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '06e81d67-f6fa-4ec8-babb-ba6eb56ef592'
2024-12-11T20:13:09,636 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-12-11T20:13:09,636 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "06e81d67-f6fa-4ec8-babb-ba6eb56ef592"

2024-12-11T20:13:09,637 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@270b75fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-11T20:13:09,637 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9fe0640122ec,40715,-1]
2024-12-11T20:13:09,637 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-12-11T20:13:09,637 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T20:13:09,641 INFO  [HMaster-EventLoopGroup-10-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56670, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-12-11T20:13:09,642 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64d3fc67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-11T20:13:09,643 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-11T20:13:09,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9fe0640122ec,45215,1733947988303, seqNum=-1]
2024-12-11T20:13:09,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-11T20:13:09,646 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54416, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-11T20:13:09,649 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=9fe0640122ec,40715,1733947988255
2024-12-11T20:13:09,649 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T20:13:09,652 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-12-11T20:13:09,652 INFO  [Time-limited test {}] wal.TestLogRolling(320): Starting testLogRollOnPipelineRestart
2024-12-11T20:13:09,652 INFO  [Time-limited test {}] wal.TestLogRolling(323): Replication=2
2024-12-11T20:13:09,653 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-12-11T20:13:09,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncConnectionImpl(321): The fetched master address is 9fe0640122ec,40715,1733947988255
2024-12-11T20:13:09,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1f48e9bc
2024-12-11T20:13:09,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-12-11T20:13:09,659 INFO  [HMaster-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56682, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-12-11T20:13:09,660 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40715 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-12-11T20:13:09,660 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40715 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-12-11T20:13:09,660 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40715 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRollOnPipelineRestart', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-11T20:13:09,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40715 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart
2024-12-11T20:13:09,664 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_PRE_OPERATION
2024-12-11T20:13:09,664 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:13:09,664 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40715 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRollOnPipelineRestart"
 procId is: 4
2024-12-11T20:13:09,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40715 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-11T20:13:09,666 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-11T20:13:09,680 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33075 is added to blk_1073741835_1011 (size=395)
2024-12-11T20:13:09,681 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37927 is added to blk_1073741835_1011 (size=395)
2024-12-11T20:13:09,683 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => bf193a2f11156da1d1b7407be96c4423, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1733947989660.bf193a2f11156da1d1b7407be96c4423.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnPipelineRestart', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3
2024-12-11T20:13:09,695 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33075 is added to blk_1073741836_1012 (size=78)
2024-12-11T20:13:09,695 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37927 is added to blk_1073741836_1012 (size=78)
2024-12-11T20:13:09,696 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1733947989660.bf193a2f11156da1d1b7407be96c4423.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T20:13:09,696 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1722): Closing bf193a2f11156da1d1b7407be96c4423, disabling compactions & flushes
2024-12-11T20:13:09,696 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1733947989660.bf193a2f11156da1d1b7407be96c4423.
2024-12-11T20:13:09,696 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1733947989660.bf193a2f11156da1d1b7407be96c4423.
2024-12-11T20:13:09,696 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1733947989660.bf193a2f11156da1d1b7407be96c4423. after waiting 0 ms
2024-12-11T20:13:09,696 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1733947989660.bf193a2f11156da1d1b7407be96c4423.
2024-12-11T20:13:09,696 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnPipelineRestart,,1733947989660.bf193a2f11156da1d1b7407be96c4423.
2024-12-11T20:13:09,696 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1676): Region close journal for bf193a2f11156da1d1b7407be96c4423:
Waiting for close lock at 1733947989696Disabling compacts and flushes for region at 1733947989696Disabling writes for close at 1733947989696Writing region close event to WAL at 1733947989696Closed at 1733947989696
2024-12-11T20:13:09,698 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ADD_TO_META
2024-12-11T20:13:09,699 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnPipelineRestart,,1733947989660.bf193a2f11156da1d1b7407be96c4423.","families":{"info":[{"qualifier":"regioninfo","vlen":77,"tag":[],"timestamp":"1733947989698"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733947989698"}]},"ts":"1733947989698"}
2024-12-11T20:13:09,702 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-12-11T20:13:09,704 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-11T20:13:09,704 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733947989704"}]},"ts":"1733947989704"}
2024-12-11T20:13:09,707 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLING in hbase:meta
2024-12-11T20:13:09,707 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=bf193a2f11156da1d1b7407be96c4423, ASSIGN}]
2024-12-11T20:13:09,709 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=bf193a2f11156da1d1b7407be96c4423, ASSIGN
2024-12-11T20:13:09,710 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=bf193a2f11156da1d1b7407be96c4423, ASSIGN; state=OFFLINE, location=9fe0640122ec,45215,1733947988303; forceNewPlan=false, retain=false
2024-12-11T20:13:09,861 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=bf193a2f11156da1d1b7407be96c4423, regionState=OPENING, regionLocation=9fe0640122ec,45215,1733947988303
2024-12-11T20:13:09,864 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-11-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=bf193a2f11156da1d1b7407be96c4423, ASSIGN because future has completed
2024-12-11T20:13:09,865 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure bf193a2f11156da1d1b7407be96c4423, server=9fe0640122ec,45215,1733947988303}]
2024-12-11T20:13:10,022 INFO  [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRollOnPipelineRestart,,1733947989660.bf193a2f11156da1d1b7407be96c4423.
2024-12-11T20:13:10,022 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => bf193a2f11156da1d1b7407be96c4423, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1733947989660.bf193a2f11156da1d1b7407be96c4423.', STARTKEY => '', ENDKEY => ''}
2024-12-11T20:13:10,023 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnPipelineRestart bf193a2f11156da1d1b7407be96c4423
2024-12-11T20:13:10,023 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1733947989660.bf193a2f11156da1d1b7407be96c4423.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T20:13:10,023 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for bf193a2f11156da1d1b7407be96c4423
2024-12-11T20:13:10,023 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for bf193a2f11156da1d1b7407be96c4423
2024-12-11T20:13:10,024 INFO  [StoreOpener-bf193a2f11156da1d1b7407be96c4423-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region bf193a2f11156da1d1b7407be96c4423 
2024-12-11T20:13:10,026 INFO  [StoreOpener-bf193a2f11156da1d1b7407be96c4423-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bf193a2f11156da1d1b7407be96c4423 columnFamilyName info
2024-12-11T20:13:10,026 DEBUG [StoreOpener-bf193a2f11156da1d1b7407be96c4423-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:13:10,026 INFO  [StoreOpener-bf193a2f11156da1d1b7407be96c4423-1 {}] regionserver.HStore(327): Store=bf193a2f11156da1d1b7407be96c4423/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-11T20:13:10,026 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for bf193a2f11156da1d1b7407be96c4423
2024-12-11T20:13:10,027 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/data/default/TestLogRolling-testLogRollOnPipelineRestart/bf193a2f11156da1d1b7407be96c4423
2024-12-11T20:13:10,027 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/data/default/TestLogRolling-testLogRollOnPipelineRestart/bf193a2f11156da1d1b7407be96c4423
2024-12-11T20:13:10,028 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for bf193a2f11156da1d1b7407be96c4423
2024-12-11T20:13:10,028 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for bf193a2f11156da1d1b7407be96c4423
2024-12-11T20:13:10,030 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for bf193a2f11156da1d1b7407be96c4423
2024-12-11T20:13:10,032 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/data/default/TestLogRolling-testLogRollOnPipelineRestart/bf193a2f11156da1d1b7407be96c4423/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-11T20:13:10,032 INFO  [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened bf193a2f11156da1d1b7407be96c4423; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=793472, jitterRate=0.008952736854553223}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-11T20:13:10,032 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bf193a2f11156da1d1b7407be96c4423
2024-12-11T20:13:10,033 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for bf193a2f11156da1d1b7407be96c4423:
Running coprocessor pre-open hook at 1733947990023Writing region info on filesystem at 1733947990023Initializing all the Stores at 1733947990024 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733947990024Cleaning up temporary data from old regions at 1733947990028 (+4 ms)Running coprocessor post-open hooks at 1733947990032 (+4 ms)Region opened successfully at 1733947990033 (+1 ms)
2024-12-11T20:13:10,034 INFO  [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRollOnPipelineRestart,,1733947989660.bf193a2f11156da1d1b7407be96c4423., pid=6, masterSystemTime=1733947990018
2024-12-11T20:13:10,036 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRollOnPipelineRestart,,1733947989660.bf193a2f11156da1d1b7407be96c4423.
2024-12-11T20:13:10,036 INFO  [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRollOnPipelineRestart,,1733947989660.bf193a2f11156da1d1b7407be96c4423.
2024-12-11T20:13:10,037 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=bf193a2f11156da1d1b7407be96c4423, regionState=OPEN, openSeqNum=2, regionLocation=9fe0640122ec,45215,1733947988303
2024-12-11T20:13:10,039 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-11-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure bf193a2f11156da1d1b7407be96c4423, server=9fe0640122ec,45215,1733947988303 because future has completed
2024-12-11T20:13:10,043 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-12-11T20:13:10,043 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure bf193a2f11156da1d1b7407be96c4423, server=9fe0640122ec,45215,1733947988303 in 176 msec
2024-12-11T20:13:10,046 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-12-11T20:13:10,046 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=bf193a2f11156da1d1b7407be96c4423, ASSIGN in 336 msec
2024-12-11T20:13:10,047 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-11T20:13:10,047 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733947990047"}]},"ts":"1733947990047"}
2024-12-11T20:13:10,049 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLED in hbase:meta
2024-12-11T20:13:10,050 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_POST_OPERATION
2024-12-11T20:13:10,052 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart in 390 msec
2024-12-11T20:13:10,266 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:10,272 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:11,267 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:11,272 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:12,156 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-12-11T20:13:12,156 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-12-11T20:13:12,156 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart
2024-12-11T20:13:12,156 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart Metrics about Tables on a single HBase RegionServer
2024-12-11T20:13:12,157 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-11T20:13:12,157 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-12-11T20:13:12,268 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:12,273 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:13,268 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:13,274 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:14,269 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:14,274 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:15,027 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-11T20:13:15,043 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:15,043 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:15,044 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:15,044 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:15,044 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:15,045 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:15,048 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:15,048 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:15,048 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:15,051 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:15,056 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-12-11T20:13:15,057 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnPipelineRestart'
2024-12-11T20:13:15,270 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:15,274 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:16,270 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:16,275 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:17,271 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:17,275 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:18,272 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:18,276 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:19,272 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:19,277 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:19,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40715 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-11T20:13:19,748 INFO  [RPCClient-NioEventLoopGroup-4-15 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnPipelineRestart completed
2024-12-11T20:13:19,748 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRollOnPipelineRestart,, stopping at row=TestLogRolling-testLogRollOnPipelineRestart ,, for max=2147483647 with caching=100
2024-12-11T20:13:19,751 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRollOnPipelineRestart
2024-12-11T20:13:19,751 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRollOnPipelineRestart,,1733947989660.bf193a2f11156da1d1b7407be96c4423.
2024-12-11T20:13:19,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRollOnPipelineRestart', row='row1002', locateType=CURRENT is [region=TestLogRolling-testLogRollOnPipelineRestart,,1733947989660.bf193a2f11156da1d1b7407be96c4423., hostname=9fe0640122ec,45215,1733947988303, seqNum=2]
2024-12-11T20:13:20,273 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:20,277 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:21,274 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:21,278 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:21,757 INFO  [Time-limited test {}] wal.TestLogRolling(360): log.getCurrentFileName()): hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.1733947989109
2024-12-11T20:13:21,757 WARN  [ResponseProcessor for block BP-558819663-172.17.0.2-1733947987520:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-558819663-172.17.0.2-1733947987520:blk_1073741834_1010
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:13:21,757 WARN  [ResponseProcessor for block BP-558819663-172.17.0.2-1733947987520:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-558819663-172.17.0.2-1733947987520:blk_1073741833_1009
java.io.IOException: Bad response ERROR for BP-558819663-172.17.0.2-1733947987520:blk_1073741833_1009 from datanode DatanodeInfoWithStorage[127.0.0.1:37927,DS-5af1d7d7-8339-47f7-888c-d7823d3eef19,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:13:21,758 WARN  [DataStreamer for file /user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.1733947989109 block BP-558819663-172.17.0.2-1733947987520:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-558819663-172.17.0.2-1733947987520:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33075,DS-7c862880-c949-4d2b-b189-775d30f1b8af,DISK], DatanodeInfoWithStorage[127.0.0.1:37927,DS-5af1d7d7-8339-47f7-888c-d7823d3eef19,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:37927,DS-5af1d7d7-8339-47f7-888c-d7823d3eef19,DISK]) is bad.
2024-12-11T20:13:21,758 WARN  [ResponseProcessor for block BP-558819663-172.17.0.2-1733947987520:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-558819663-172.17.0.2-1733947987520:blk_1073741830_1006
java.io.IOException: Bad response ERROR for BP-558819663-172.17.0.2-1733947987520:blk_1073741830_1006 from datanode DatanodeInfoWithStorage[127.0.0.1:37927,DS-5af1d7d7-8339-47f7-888c-d7823d3eef19,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:13:21,758 WARN  [DataStreamer for file /user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/MasterData/WALs/9fe0640122ec,40715,1733947988255/9fe0640122ec%2C40715%2C1733947988255.1733947988844 block BP-558819663-172.17.0.2-1733947987520:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-558819663-172.17.0.2-1733947987520:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33075,DS-7c862880-c949-4d2b-b189-775d30f1b8af,DISK], DatanodeInfoWithStorage[127.0.0.1:37927,DS-5af1d7d7-8339-47f7-888c-d7823d3eef19,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:37927,DS-5af1d7d7-8339-47f7-888c-d7823d3eef19,DISK]) is bad.
2024-12-11T20:13:21,758 WARN  [PacketResponder: BP-558819663-172.17.0.2-1733947987520:blk_1073741833_1009, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:37927] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:13:21,758 WARN  [PacketResponder: BP-558819663-172.17.0.2-1733947987520:blk_1073741830_1006, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:37927] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:13:21,758 WARN  [DataStreamer for file /user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.meta.1733947989516.meta block BP-558819663-172.17.0.2-1733947987520:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-558819663-172.17.0.2-1733947987520:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37927,DS-5af1d7d7-8339-47f7-888c-d7823d3eef19,DISK], DatanodeInfoWithStorage[127.0.0.1:33075,DS-7c862880-c949-4d2b-b189-775d30f1b8af,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37927,DS-5af1d7d7-8339-47f7-888c-d7823d3eef19,DISK]) is bad.
2024-12-11T20:13:21,759 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-139625735_22 at /127.0.0.1:35672 [Receiving block BP-558819663-172.17.0.2-1733947987520:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:33075:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35672 dst: /127.0.0.1:33075
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:13:21,759 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_541903910_22 at /127.0.0.1:35648 [Receiving block BP-558819663-172.17.0.2-1733947987520:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:33075:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35648 dst: /127.0.0.1:33075
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:13:21,759 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-139625735_22 at /127.0.0.1:46212 [Receiving block BP-558819663-172.17.0.2-1733947987520:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:37927:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:46212 dst: /127.0.0.1:37927
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:13:21,759 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_541903910_22 at /127.0.0.1:46170 [Receiving block BP-558819663-172.17.0.2-1733947987520:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:37927:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:46170 dst: /127.0.0.1:37927
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:13:21,759 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-139625735_22 at /127.0.0.1:35688 [Receiving block BP-558819663-172.17.0.2-1733947987520:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:33075:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35688 dst: /127.0.0.1:33075
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:13:21,759 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-139625735_22 at /127.0.0.1:46198 [Receiving block BP-558819663-172.17.0.2-1733947987520:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:37927:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:46198 dst: /127.0.0.1:37927
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:13:21,761 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6d74671d{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T20:13:21,762 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4247f9cf{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-11T20:13:21,762 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-11T20:13:21,762 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@44bd8b73{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-11T20:13:21,762 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@33392a77{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6761f64-4d8d-e744-edc5-9c12798c8e6a/hadoop.log.dir/,STOPPED}
2024-12-11T20:13:21,764 WARN  [BP-558819663-172.17.0.2-1733947987520 heartbeating to localhost/127.0.0.1:42243 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-11T20:13:21,764 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-11T20:13:21,764 WARN  [BP-558819663-172.17.0.2-1733947987520 heartbeating to localhost/127.0.0.1:42243 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-558819663-172.17.0.2-1733947987520 (Datanode Uuid c24dc4a7-9b08-451a-82ef-036733e0d594) service to localhost/127.0.0.1:42243
2024-12-11T20:13:21,764 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-11T20:13:21,765 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6761f64-4d8d-e744-edc5-9c12798c8e6a/cluster_e6efd374-2de3-88d0-f464-6076ba3d414f/data/data3/current/BP-558819663-172.17.0.2-1733947987520 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T20:13:21,765 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6761f64-4d8d-e744-edc5-9c12798c8e6a/cluster_e6efd374-2de3-88d0-f464-6076ba3d414f/data/data4/current/BP-558819663-172.17.0.2-1733947987520 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T20:13:21,765 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-11T20:13:21,774 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T20:13:21,777 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-11T20:13:21,778 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-11T20:13:21,778 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-11T20:13:21,778 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-11T20:13:21,778 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1a8d3987{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6761f64-4d8d-e744-edc5-9c12798c8e6a/hadoop.log.dir/,AVAILABLE}
2024-12-11T20:13:21,779 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5be81ff8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-11T20:13:21,895 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3237c5f1{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6761f64-4d8d-e744-edc5-9c12798c8e6a/java.io.tmpdir/jetty-localhost-34389-hadoop-hdfs-3_4_1-tests_jar-_-any-16204151405483466962/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T20:13:21,895 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4d4fe11a{HTTP/1.1, (http/1.1)}{localhost:34389}
2024-12-11T20:13:21,895 INFO  [Time-limited test {}] server.Server(415): Started @165491ms
2024-12-11T20:13:21,896 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-11T20:13:21,914 WARN  [ResponseProcessor for block BP-558819663-172.17.0.2-1733947987520:blk_1073741830_1013 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-558819663-172.17.0.2-1733947987520:blk_1073741830_1013
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:13:21,914 WARN  [ResponseProcessor for block BP-558819663-172.17.0.2-1733947987520:blk_1073741833_1014 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-558819663-172.17.0.2-1733947987520:blk_1073741833_1014
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:13:21,914 WARN  [ResponseProcessor for block BP-558819663-172.17.0.2-1733947987520:blk_1073741834_1015 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-558819663-172.17.0.2-1733947987520:blk_1073741834_1015
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:13:21,914 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-139625735_22 at /127.0.0.1:33404 [Receiving block BP-558819663-172.17.0.2-1733947987520:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:33075:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:33404 dst: /127.0.0.1:33075
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:13:21,914 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_541903910_22 at /127.0.0.1:33390 [Receiving block BP-558819663-172.17.0.2-1733947987520:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:33075:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:33390 dst: /127.0.0.1:33075
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:13:21,915 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-139625735_22 at /127.0.0.1:33414 [Receiving block BP-558819663-172.17.0.2-1733947987520:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:33075:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:33414 dst: /127.0.0.1:33075
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:13:21,922 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@624b3986{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T20:13:21,922 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7e13d66c{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-11T20:13:21,922 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-11T20:13:21,922 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@32d7b29f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-11T20:13:21,923 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@54562211{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6761f64-4d8d-e744-edc5-9c12798c8e6a/hadoop.log.dir/,STOPPED}
2024-12-11T20:13:21,924 WARN  [BP-558819663-172.17.0.2-1733947987520 heartbeating to localhost/127.0.0.1:42243 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-11T20:13:21,925 WARN  [BP-558819663-172.17.0.2-1733947987520 heartbeating to localhost/127.0.0.1:42243 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-558819663-172.17.0.2-1733947987520 (Datanode Uuid 4c3e95e7-fe9c-4d12-ba2c-9bb9508f688f) service to localhost/127.0.0.1:42243
2024-12-11T20:13:21,925 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-11T20:13:21,925 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-11T20:13:21,925 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6761f64-4d8d-e744-edc5-9c12798c8e6a/cluster_e6efd374-2de3-88d0-f464-6076ba3d414f/data/data1/current/BP-558819663-172.17.0.2-1733947987520 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T20:13:21,926 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6761f64-4d8d-e744-edc5-9c12798c8e6a/cluster_e6efd374-2de3-88d0-f464-6076ba3d414f/data/data2/current/BP-558819663-172.17.0.2-1733947987520 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T20:13:21,926 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-11T20:13:21,933 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T20:13:21,937 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-11T20:13:21,937 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-11T20:13:21,937 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-11T20:13:21,937 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-11T20:13:21,938 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7cbeb460{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6761f64-4d8d-e744-edc5-9c12798c8e6a/hadoop.log.dir/,AVAILABLE}
2024-12-11T20:13:21,938 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3a7e8687{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-11T20:13:21,980 WARN  [Thread-1343 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-11T20:13:21,983 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4e8f7321c90d91f7 with lease ID 0xa43fd1a80235cb3a: from storage DS-5af1d7d7-8339-47f7-888c-d7823d3eef19 node DatanodeRegistration(127.0.0.1:35283, datanodeUuid=c24dc4a7-9b08-451a-82ef-036733e0d594, infoPort=37941, infoSecurePort=0, ipcPort=43507, storageInfo=lv=-57;cid=testClusterID;nsid=1183363567;c=1733947987520), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T20:13:21,983 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4e8f7321c90d91f7 with lease ID 0xa43fd1a80235cb3a: from storage DS-c33d6bb7-cece-417c-82dd-c95497ef7faf node DatanodeRegistration(127.0.0.1:35283, datanodeUuid=c24dc4a7-9b08-451a-82ef-036733e0d594, infoPort=37941, infoSecurePort=0, ipcPort=43507, storageInfo=lv=-57;cid=testClusterID;nsid=1183363567;c=1733947987520), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T20:13:22,061 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@55a0f64a{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6761f64-4d8d-e744-edc5-9c12798c8e6a/java.io.tmpdir/jetty-localhost-42585-hadoop-hdfs-3_4_1-tests_jar-_-any-17324733932903061019/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T20:13:22,061 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3bfe6512{HTTP/1.1, (http/1.1)}{localhost:42585}
2024-12-11T20:13:22,061 INFO  [Time-limited test {}] server.Server(415): Started @165657ms
2024-12-11T20:13:22,062 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-11T20:13:22,141 WARN  [Thread-1374 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-11T20:13:22,144 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa138c105496bbd67 with lease ID 0xa43fd1a80235cb3b: from storage DS-7c862880-c949-4d2b-b189-775d30f1b8af node DatanodeRegistration(127.0.0.1:40295, datanodeUuid=4c3e95e7-fe9c-4d12-ba2c-9bb9508f688f, infoPort=42103, infoSecurePort=0, ipcPort=41525, storageInfo=lv=-57;cid=testClusterID;nsid=1183363567;c=1733947987520), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T20:13:22,144 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa138c105496bbd67 with lease ID 0xa43fd1a80235cb3b: from storage DS-3ff62e98-e67b-4e4c-a8eb-73b19b99f495 node DatanodeRegistration(127.0.0.1:40295, datanodeUuid=4c3e95e7-fe9c-4d12-ba2c-9bb9508f688f, infoPort=42103, infoSecurePort=0, ipcPort=41525, storageInfo=lv=-57;cid=testClusterID;nsid=1183363567;c=1733947987520), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T20:13:22,274 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:22,278 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:23,080 INFO  [Time-limited test {}] wal.TestLogRolling(372): Data Nodes restarted
2024-12-11T20:13:23,082 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1002
2024-12-11T20:13:23,084 ERROR [FSHLog-0-hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3-prefix:9fe0640122ec,45215,1733947988303 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33075,DS-7c862880-c949-4d2b-b189-775d30f1b8af,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:13:23,084 WARN  [FSHLog-0-hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3-prefix:9fe0640122ec,45215,1733947988303 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33075,DS-7c862880-c949-4d2b-b189-775d30f1b8af,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:13:23,084 DEBUG [regionserver/9fe0640122ec:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 9fe0640122ec%2C45215%2C1733947988303:(num 1733947989109) roll requested
2024-12-11T20:13:23,084 INFO  [regionserver/9fe0640122ec:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9fe0640122ec%2C45215%2C1733947988303.1733948003084
2024-12-11T20:13:23,090 DEBUG [regionserver/9fe0640122ec:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.1733947989109 newFile=hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.1733948003084
2024-12-11T20:13:23,090 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:23,090 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:23,090 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:23,090 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:23,091 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:23,091 INFO  [regionserver/9fe0640122ec:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.1733947989109 with entries=2, filesize=1.59 KB; new WAL /user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.1733948003084
2024-12-11T20:13:23,091 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33075,DS-7c862880-c949-4d2b-b189-775d30f1b8af,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:13:23,091 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33075,DS-7c862880-c949-4d2b-b189-775d30f1b8af,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:13:23,091 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.1733947989109
2024-12-11T20:13:23,092 DEBUG [regionserver/9fe0640122ec:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37941:37941),(127.0.0.1/127.0.0.1:42103:42103)]
2024-12-11T20:13:23,092 DEBUG [regionserver/9fe0640122ec:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.1733947989109 is not closed yet, will try archiving it next time
2024-12-11T20:13:23,092 WARN  [IPC Server handler 2 on default port 42243 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.1733947989109 has not been closed. Lease recovery is in progress. RecoveryId = 1017 for block blk_1073741833_1014
2024-12-11T20:13:23,092 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.1733947989109 after 1ms
2024-12-11T20:13:23,275 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:23,279 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:24,275 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:24,279 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:25,095 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1003
2024-12-11T20:13:25,276 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:25,279 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:25,983 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741833_1014: GenerationStamp not matched, existing replica is blk_1073741833_1009
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-12-11T20:13:26,277 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:26,280 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:27,095 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.1733947989109 after 4004ms
2024-12-11T20:13:27,098 WARN  [ResponseProcessor for block BP-558819663-172.17.0.2-1733947987520:blk_1073741837_1016 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-558819663-172.17.0.2-1733947987520:blk_1073741837_1016
java.io.IOException: Bad response ERROR for BP-558819663-172.17.0.2-1733947987520:blk_1073741837_1016 from datanode DatanodeInfoWithStorage[127.0.0.1:40295,DS-7c862880-c949-4d2b-b189-775d30f1b8af,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:13:27,098 WARN  [DataStreamer for file /user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.1733948003084 block BP-558819663-172.17.0.2-1733947987520:blk_1073741837_1016 {}] hdfs.DataStreamer(1731): Error Recovery for BP-558819663-172.17.0.2-1733947987520:blk_1073741837_1016 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35283,DS-5af1d7d7-8339-47f7-888c-d7823d3eef19,DISK], DatanodeInfoWithStorage[127.0.0.1:40295,DS-7c862880-c949-4d2b-b189-775d30f1b8af,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:40295,DS-7c862880-c949-4d2b-b189-775d30f1b8af,DISK]) is bad.
2024-12-11T20:13:27,098 WARN  [PacketResponder: BP-558819663-172.17.0.2-1733947987520:blk_1073741837_1016, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:40295] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:13:27,099 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-139625735_22 at /127.0.0.1:51180 [Receiving block BP-558819663-172.17.0.2-1733947987520:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:35283:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:51180 dst: /127.0.0.1:35283
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:13:27,099 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-139625735_22 at /127.0.0.1:38436 [Receiving block BP-558819663-172.17.0.2-1733947987520:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:40295:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:38436 dst: /127.0.0.1:40295
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:13:27,100 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@55a0f64a{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T20:13:27,101 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3bfe6512{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-11T20:13:27,101 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-11T20:13:27,101 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3a7e8687{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-11T20:13:27,101 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7cbeb460{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6761f64-4d8d-e744-edc5-9c12798c8e6a/hadoop.log.dir/,STOPPED}
2024-12-11T20:13:27,102 WARN  [BP-558819663-172.17.0.2-1733947987520 heartbeating to localhost/127.0.0.1:42243 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-11T20:13:27,102 WARN  [BP-558819663-172.17.0.2-1733947987520 heartbeating to localhost/127.0.0.1:42243 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-558819663-172.17.0.2-1733947987520 (Datanode Uuid 4c3e95e7-fe9c-4d12-ba2c-9bb9508f688f) service to localhost/127.0.0.1:42243
2024-12-11T20:13:27,102 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-11T20:13:27,103 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-11T20:13:27,103 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6761f64-4d8d-e744-edc5-9c12798c8e6a/cluster_e6efd374-2de3-88d0-f464-6076ba3d414f/data/data1/current/BP-558819663-172.17.0.2-1733947987520 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T20:13:27,103 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6761f64-4d8d-e744-edc5-9c12798c8e6a/cluster_e6efd374-2de3-88d0-f464-6076ba3d414f/data/data2/current/BP-558819663-172.17.0.2-1733947987520 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T20:13:27,104 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-11T20:13:27,114 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T20:13:27,117 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-11T20:13:27,118 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-11T20:13:27,118 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-11T20:13:27,118 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-11T20:13:27,118 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@78cc0925{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6761f64-4d8d-e744-edc5-9c12798c8e6a/hadoop.log.dir/,AVAILABLE}
2024-12-11T20:13:27,119 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@731665df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-11T20:13:27,235 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2026736d{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6761f64-4d8d-e744-edc5-9c12798c8e6a/java.io.tmpdir/jetty-localhost-43475-hadoop-hdfs-3_4_1-tests_jar-_-any-17844373087435259957/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T20:13:27,235 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@22b034d5{HTTP/1.1, (http/1.1)}{localhost:43475}
2024-12-11T20:13:27,235 INFO  [Time-limited test {}] server.Server(415): Started @170831ms
2024-12-11T20:13:27,237 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-11T20:13:27,255 WARN  [ResponseProcessor for block BP-558819663-172.17.0.2-1733947987520:blk_1073741837_1018 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-558819663-172.17.0.2-1733947987520:blk_1073741837_1018
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:13:27,255 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-139625735_22 at /127.0.0.1:54624 [Receiving block BP-558819663-172.17.0.2-1733947987520:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:35283:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:54624 dst: /127.0.0.1:35283
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:13:27,257 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3237c5f1{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T20:13:27,258 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4d4fe11a{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-11T20:13:27,258 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-11T20:13:27,258 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5be81ff8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-11T20:13:27,258 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1a8d3987{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6761f64-4d8d-e744-edc5-9c12798c8e6a/hadoop.log.dir/,STOPPED}
2024-12-11T20:13:27,259 WARN  [BP-558819663-172.17.0.2-1733947987520 heartbeating to localhost/127.0.0.1:42243 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-11T20:13:27,259 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-11T20:13:27,259 WARN  [BP-558819663-172.17.0.2-1733947987520 heartbeating to localhost/127.0.0.1:42243 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-558819663-172.17.0.2-1733947987520 (Datanode Uuid c24dc4a7-9b08-451a-82ef-036733e0d594) service to localhost/127.0.0.1:42243
2024-12-11T20:13:27,260 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-11T20:13:27,261 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6761f64-4d8d-e744-edc5-9c12798c8e6a/cluster_e6efd374-2de3-88d0-f464-6076ba3d414f/data/data3/current/BP-558819663-172.17.0.2-1733947987520 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T20:13:27,261 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6761f64-4d8d-e744-edc5-9c12798c8e6a/cluster_e6efd374-2de3-88d0-f464-6076ba3d414f/data/data4/current/BP-558819663-172.17.0.2-1733947987520 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T20:13:27,261 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-11T20:13:27,270 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T20:13:27,273 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-11T20:13:27,274 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-11T20:13:27,274 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-11T20:13:27,274 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-11T20:13:27,274 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4f7ae33f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6761f64-4d8d-e744-edc5-9c12798c8e6a/hadoop.log.dir/,AVAILABLE}
2024-12-11T20:13:27,275 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@bc30fa1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-11T20:13:27,277 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:27,280 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:27,343 WARN  [Thread-1417 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-11T20:13:27,346 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6af0f112109e6e45 with lease ID 0xa43fd1a80235cb3c: from storage DS-7c862880-c949-4d2b-b189-775d30f1b8af node DatanodeRegistration(127.0.0.1:40081, datanodeUuid=4c3e95e7-fe9c-4d12-ba2c-9bb9508f688f, infoPort=36151, infoSecurePort=0, ipcPort=46039, storageInfo=lv=-57;cid=testClusterID;nsid=1183363567;c=1733947987520), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T20:13:27,346 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6af0f112109e6e45 with lease ID 0xa43fd1a80235cb3c: from storage DS-3ff62e98-e67b-4e4c-a8eb-73b19b99f495 node DatanodeRegistration(127.0.0.1:40081, datanodeUuid=4c3e95e7-fe9c-4d12-ba2c-9bb9508f688f, infoPort=36151, infoSecurePort=0, ipcPort=46039, storageInfo=lv=-57;cid=testClusterID;nsid=1183363567;c=1733947987520), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T20:13:27,393 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@77ebbd9c{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6761f64-4d8d-e744-edc5-9c12798c8e6a/java.io.tmpdir/jetty-localhost-35275-hadoop-hdfs-3_4_1-tests_jar-_-any-14190699538805020608/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T20:13:27,393 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@14b40a8a{HTTP/1.1, (http/1.1)}{localhost:35275}
2024-12-11T20:13:27,393 INFO  [Time-limited test {}] server.Server(415): Started @170989ms
2024-12-11T20:13:27,395 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-11T20:13:27,485 WARN  [Thread-1448 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-11T20:13:27,488 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xdc81d87469e27d59 with lease ID 0xa43fd1a80235cb3d: from storage DS-5af1d7d7-8339-47f7-888c-d7823d3eef19 node DatanodeRegistration(127.0.0.1:46799, datanodeUuid=c24dc4a7-9b08-451a-82ef-036733e0d594, infoPort=43617, infoSecurePort=0, ipcPort=37599, storageInfo=lv=-57;cid=testClusterID;nsid=1183363567;c=1733947987520), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T20:13:27,488 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xdc81d87469e27d59 with lease ID 0xa43fd1a80235cb3d: from storage DS-c33d6bb7-cece-417c-82dd-c95497ef7faf node DatanodeRegistration(127.0.0.1:46799, datanodeUuid=c24dc4a7-9b08-451a-82ef-036733e0d594, infoPort=43617, infoSecurePort=0, ipcPort=37599, storageInfo=lv=-57;cid=testClusterID;nsid=1183363567;c=1733947987520), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T20:13:28,278 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:28,281 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:28,414 INFO  [Time-limited test {}] wal.TestLogRolling(389): Data Nodes restarted
2024-12-11T20:13:28,417 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1004
2024-12-11T20:13:28,418 ERROR [FSHLog-0-hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3-prefix:9fe0640122ec,45215,1733947988303 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:35283,DS-5af1d7d7-8339-47f7-888c-d7823d3eef19,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:13:28,418 WARN  [FSHLog-0-hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3-prefix:9fe0640122ec,45215,1733947988303 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:35283,DS-5af1d7d7-8339-47f7-888c-d7823d3eef19,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:13:28,418 DEBUG [regionserver/9fe0640122ec:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 9fe0640122ec%2C45215%2C1733947988303:(num 1733948003084) roll requested
2024-12-11T20:13:28,419 INFO  [regionserver/9fe0640122ec:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9fe0640122ec%2C45215%2C1733947988303.1733948008418
2024-12-11T20:13:28,425 DEBUG [regionserver/9fe0640122ec:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.1733948003084 newFile=hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.1733948008418
2024-12-11T20:13:28,425 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:28,425 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:28,425 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:28,425 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:28,426 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:28,426 INFO  [regionserver/9fe0640122ec:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.1733948003084 with entries=2, filesize=2.37 KB; new WAL /user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.1733948008418
2024-12-11T20:13:28,426 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:35283,DS-5af1d7d7-8339-47f7-888c-d7823d3eef19,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:13:28,426 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:35283,DS-5af1d7d7-8339-47f7-888c-d7823d3eef19,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:13:28,426 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.1733948003084
2024-12-11T20:13:28,427 WARN  [IPC Server handler 3 on default port 42243 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.1733948003084 has not been closed. Lease recovery is in progress. RecoveryId = 1020 for block blk_1073741837_1018
2024-12-11T20:13:28,427 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.1733948003084 after 1ms
2024-12-11T20:13:28,429 DEBUG [regionserver/9fe0640122ec:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43617:43617),(127.0.0.1/127.0.0.1:36151:36151)]
2024-12-11T20:13:28,429 DEBUG [regionserver/9fe0640122ec:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.1733948003084 is not closed yet, will try archiving it next time
2024-12-11T20:13:29,279 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:29,281 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:29,347 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741837_1018: GenerationStamp not matched, existing replica is blk_1073741837_1016
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-12-11T20:13:30,279 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:30,282 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:30,430 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9fe0640122ec%2C45215%2C1733947988303.1733948010430
2024-12-11T20:13:30,436 DEBUG [Time-limited test {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.1733948008418 newFile=hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.1733948010430
2024-12-11T20:13:30,436 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:30,436 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:30,437 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:30,437 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:30,437 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:30,437 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.1733948008418 with entries=1, filesize=1.23 KB; new WAL /user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.1733948010430
2024-12-11T20:13:30,439 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43617:43617),(127.0.0.1/127.0.0.1:36151:36151)]
2024-12-11T20:13:30,439 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.1733948003084 is not closed yet, will try archiving it next time
2024-12-11T20:13:30,439 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.1733948008418 is not closed yet, will try archiving it next time
2024-12-11T20:13:30,439 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.1733947989109
2024-12-11T20:13:30,439 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.1733947989109
2024-12-11T20:13:30,439 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40081 is added to blk_1073741838_1019 (size=1264)
2024-12-11T20:13:30,440 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.1733947989109 after 1ms
2024-12-11T20:13:30,440 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.1733947989109
2024-12-11T20:13:30,440 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46799 is added to blk_1073741838_1019 (size=1264)
2024-12-11T20:13:30,440 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.1733948003084 is not closed yet, will try archiving it next time
2024-12-11T20:13:30,450 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #3: [\x00/METAFAMILY:HBASE::REGION_EVENT::REGION_OPEN/1733947990033/Put/vlen=218/seqid=0]
2024-12-11T20:13:30,451 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #4: [row1002/info:/1733947999755/Put/vlen=1045/seqid=0]
2024-12-11T20:13:30,451 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.1733947989109
2024-12-11T20:13:30,451 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.1733948003084
2024-12-11T20:13:30,451 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.1733948003084
2024-12-11T20:13:30,451 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.1733948003084 after 0ms
2024-12-11T20:13:30,451 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.1733948003084
2024-12-11T20:13:30,455 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #5: [row1003/info:/1733948003083/Put/vlen=1045/seqid=0]
2024-12-11T20:13:30,455 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #6: [row1004/info:/1733948005096/Put/vlen=1045/seqid=0]
2024-12-11T20:13:30,455 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.1733948003084
2024-12-11T20:13:30,455 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.1733948008418
2024-12-11T20:13:30,455 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.1733948008418
2024-12-11T20:13:30,456 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.1733948008418 after 1ms
2024-12-11T20:13:30,456 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.1733948008418
2024-12-11T20:13:30,459 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #7: [row1005/info:/1733948008418/Put/vlen=1045/seqid=0]
2024-12-11T20:13:30,459 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.1733948010430
2024-12-11T20:13:30,459 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.1733948010430
2024-12-11T20:13:30,459 WARN  [IPC Server handler 1 on default port 42243 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.1733948010430 has not been closed. Lease recovery is in progress. RecoveryId = 1022 for block blk_1073741839_1021
2024-12-11T20:13:30,459 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.1733948010430 after 0ms
2024-12-11T20:13:31,280 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:31,282 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:31,350 WARN  [ResponseProcessor for block BP-558819663-172.17.0.2-1733947987520:blk_1073741839_1021 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-558819663-172.17.0.2-1733947987520:blk_1073741839_1021
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:13:31,350 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_541903910_22 at /127.0.0.1:34036 [Receiving block BP-558819663-172.17.0.2-1733947987520:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:46799:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:34036 dst: /127.0.0.1:46799
java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:46799 remote=/127.0.0.1:34036]. Total timeout mills is 60000, 59085 millis timeout left.
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:13:31,350 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_541903910_22 at /127.0.0.1:39476 [Receiving block BP-558819663-172.17.0.2-1733947987520:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:40081:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:39476 dst: /127.0.0.1:40081
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:13:31,351 WARN  [DataStreamer for file /user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.1733948010430 block BP-558819663-172.17.0.2-1733947987520:blk_1073741839_1021 {}] hdfs.DataStreamer(1731): Error Recovery for BP-558819663-172.17.0.2-1733947987520:blk_1073741839_1021 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46799,DS-5af1d7d7-8339-47f7-888c-d7823d3eef19,DISK], DatanodeInfoWithStorage[127.0.0.1:40081,DS-7c862880-c949-4d2b-b189-775d30f1b8af,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:46799,DS-5af1d7d7-8339-47f7-888c-d7823d3eef19,DISK]) is bad.
2024-12-11T20:13:31,352 WARN  [DataStreamer for file /user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.1733948010430 block BP-558819663-172.17.0.2-1733947987520:blk_1073741839_1021 {}] hdfs.DataStreamer(859): DataStreamer Exception
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-558819663-172.17.0.2-1733947987520:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:13:31,356 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46799 is added to blk_1073741839_1022 (size=85)
2024-12-11T20:13:32,281 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:32,283 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:32,428 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.1733948003084 after 4002ms
2024-12-11T20:13:33,281 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:33,283 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:34,282 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:34,284 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:34,460 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.1733948010430 after 4001ms
2024-12-11T20:13:34,460 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.1733948010430
2024-12-11T20:13:34,464 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.1733948010430
2024-12-11T20:13:34,464 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing bf193a2f11156da1d1b7407be96c4423 1/1 column families, dataSize=4.20 KB heapSize=4.75 KB
2024-12-11T20:13:34,465 ERROR [FSHLog-0-hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3-prefix:9fe0640122ec,45215,1733947988303 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-558819663-172.17.0.2-1733947987520:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:13:34,465 WARN  [FSHLog-0-hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3-prefix:9fe0640122ec,45215,1733947988303 {}] wal.AbstractFSWAL(2174): append entry failed
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-558819663-172.17.0.2-1733947987520:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:13:34,465 DEBUG [regionserver/9fe0640122ec:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 9fe0640122ec%2C45215%2C1733947988303:(num 1733948010430) roll requested
2024-12-11T20:13:34,466 INFO  [regionserver/9fe0640122ec:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9fe0640122ec%2C45215%2C1733947988303.1733948014466
2024-12-11T20:13:34,472 DEBUG [regionserver/9fe0640122ec:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.1733948010430 newFile=hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.1733948014466
2024-12-11T20:13:34,473 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:34,473 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:34,473 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:34,473 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:34,473 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:34,473 INFO  [regionserver/9fe0640122ec:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.1733948010430 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.1733948014466
2024-12-11T20:13:34,473 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-558819663-172.17.0.2-1733947987520:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:13:34,474 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-558819663-172.17.0.2-1733947987520:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:13:34,474 DEBUG [regionserver/9fe0640122ec:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43617:43617),(127.0.0.1/127.0.0.1:36151:36151)]
2024-12-11T20:13:34,474 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.1733948010430
2024-12-11T20:13:34,474 DEBUG [regionserver/9fe0640122ec:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.1733948010430 is not closed yet, will try archiving it next time
2024-12-11T20:13:34,474 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.1733948010430 after 0ms
2024-12-11T20:13:34,475 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.1733948010430 to hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/oldWALs/9fe0640122ec%2C45215%2C1733947988303.1733948010430
2024-12-11T20:13:34,493 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/data/default/TestLogRolling-testLogRollOnPipelineRestart/bf193a2f11156da1d1b7407be96c4423/.tmp/info/03e66ce7a3b14a67b8e339c22e446c3f is 1080, key is row1002/info:/1733947999755/Put/seqid=0
2024-12-11T20:13:34,498 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40081 is added to blk_1073741841_1024 (size=9270)
2024-12-11T20:13:34,498 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46799 is added to blk_1073741841_1024 (size=9270)
2024-12-11T20:13:34,498 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.20 KB at sequenceid=8 (bloomFilter=true), to=hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/data/default/TestLogRolling-testLogRollOnPipelineRestart/bf193a2f11156da1d1b7407be96c4423/.tmp/info/03e66ce7a3b14a67b8e339c22e446c3f
2024-12-11T20:13:34,505 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/data/default/TestLogRolling-testLogRollOnPipelineRestart/bf193a2f11156da1d1b7407be96c4423/.tmp/info/03e66ce7a3b14a67b8e339c22e446c3f as hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/data/default/TestLogRolling-testLogRollOnPipelineRestart/bf193a2f11156da1d1b7407be96c4423/info/03e66ce7a3b14a67b8e339c22e446c3f
2024-12-11T20:13:34,510 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/data/default/TestLogRolling-testLogRollOnPipelineRestart/bf193a2f11156da1d1b7407be96c4423/info/03e66ce7a3b14a67b8e339c22e446c3f, entries=4, sequenceid=8, filesize=9.1 K
2024-12-11T20:13:34,511 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~4.20 KB/4304, heapSize ~4.73 KB/4848, currentSize=0 B/0 for bf193a2f11156da1d1b7407be96c4423 in 47ms, sequenceid=8, compaction requested=false
2024-12-11T20:13:34,511 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for bf193a2f11156da1d1b7407be96c4423:

2024-12-11T20:13:34,511 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.74 KB heapSize=3.77 KB
2024-12-11T20:13:34,512 ERROR [FSHLog-0-hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3-prefix:9fe0640122ec,45215,1733947988303.meta {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33075,DS-7c862880-c949-4d2b-b189-775d30f1b8af,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:13:34,512 WARN  [FSHLog-0-hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3-prefix:9fe0640122ec,45215,1733947988303.meta {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33075,DS-7c862880-c949-4d2b-b189-775d30f1b8af,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:13:34,512 DEBUG [regionserver/9fe0640122ec:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 9fe0640122ec%2C45215%2C1733947988303.meta:.meta(num 1733947989516) roll requested
2024-12-11T20:13:34,512 INFO  [regionserver/9fe0640122ec:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9fe0640122ec%2C45215%2C1733947988303.meta.1733948014512.meta
2024-12-11T20:13:34,517 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:34,517 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:34,517 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:34,517 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:34,517 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:34,518 INFO  [regionserver/9fe0640122ec:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.meta.1733947989516.meta with entries=8, filesize=2.36 KB; new WAL /user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.meta.1733948014512.meta
2024-12-11T20:13:34,518 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33075,DS-7c862880-c949-4d2b-b189-775d30f1b8af,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:13:34,519 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33075,DS-7c862880-c949-4d2b-b189-775d30f1b8af,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:13:34,519 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.meta.1733947989516.meta
2024-12-11T20:13:34,519 WARN  [IPC Server handler 0 on default port 42243 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.meta.1733947989516.meta has not been closed. Lease recovery is in progress. RecoveryId = 1026 for block blk_1073741834_1015
2024-12-11T20:13:34,519 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.meta.1733947989516.meta after 0ms
2024-12-11T20:13:34,521 DEBUG [regionserver/9fe0640122ec:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36151:36151),(127.0.0.1/127.0.0.1:43617:43617)]
2024-12-11T20:13:34,521 DEBUG [regionserver/9fe0640122ec:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.meta.1733947989516.meta is not closed yet, will try archiving it next time
2024-12-11T20:13:34,536 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/data/hbase/meta/1588230740/.tmp/info/b5f3f05b1ef944cc8a9e850cfa74a7a0 is 207, key is TestLogRolling-testLogRollOnPipelineRestart,,1733947989660.bf193a2f11156da1d1b7407be96c4423./info:regioninfo/1733947990037/Put/seqid=0
2024-12-11T20:13:34,541 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46799 is added to blk_1073741843_1027 (size=7125)
2024-12-11T20:13:34,541 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40081 is added to blk_1073741843_1027 (size=7125)
2024-12-11T20:13:34,541 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.52 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/data/hbase/meta/1588230740/.tmp/info/b5f3f05b1ef944cc8a9e850cfa74a7a0
2024-12-11T20:13:34,561 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/data/hbase/meta/1588230740/.tmp/ns/2f62bccf23f84e328134c77657e603f4 is 43, key is default/ns:d/1733947989566/Put/seqid=0
2024-12-11T20:13:34,566 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40081 is added to blk_1073741844_1028 (size=5153)
2024-12-11T20:13:34,566 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46799 is added to blk_1073741844_1028 (size=5153)
2024-12-11T20:13:34,566 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/data/hbase/meta/1588230740/.tmp/ns/2f62bccf23f84e328134c77657e603f4
2024-12-11T20:13:34,586 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/data/hbase/meta/1588230740/.tmp/table/0173927071f44f13b7619fbf7e36df7b is 79, key is TestLogRolling-testLogRollOnPipelineRestart/table:state/1733947990047/Put/seqid=0
2024-12-11T20:13:34,590 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46799 is added to blk_1073741845_1029 (size=5438)
2024-12-11T20:13:34,591 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40081 is added to blk_1073741845_1029 (size=5438)
2024-12-11T20:13:34,591 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=150 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/data/hbase/meta/1588230740/.tmp/table/0173927071f44f13b7619fbf7e36df7b
2024-12-11T20:13:34,597 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/data/hbase/meta/1588230740/.tmp/info/b5f3f05b1ef944cc8a9e850cfa74a7a0 as hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/data/hbase/meta/1588230740/info/b5f3f05b1ef944cc8a9e850cfa74a7a0
2024-12-11T20:13:34,602 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/data/hbase/meta/1588230740/info/b5f3f05b1ef944cc8a9e850cfa74a7a0, entries=10, sequenceid=11, filesize=7.0 K
2024-12-11T20:13:34,603 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/data/hbase/meta/1588230740/.tmp/ns/2f62bccf23f84e328134c77657e603f4 as hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/data/hbase/meta/1588230740/ns/2f62bccf23f84e328134c77657e603f4
2024-12-11T20:13:34,608 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/data/hbase/meta/1588230740/ns/2f62bccf23f84e328134c77657e603f4, entries=2, sequenceid=11, filesize=5.0 K
2024-12-11T20:13:34,609 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/data/hbase/meta/1588230740/.tmp/table/0173927071f44f13b7619fbf7e36df7b as hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/data/hbase/meta/1588230740/table/0173927071f44f13b7619fbf7e36df7b
2024-12-11T20:13:34,614 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/data/hbase/meta/1588230740/table/0173927071f44f13b7619fbf7e36df7b, entries=2, sequenceid=11, filesize=5.3 K
2024-12-11T20:13:34,615 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.74 KB/1782, heapSize ~3.48 KB/3560, currentSize=0 B/0 for 1588230740 in 104ms, sequenceid=11, compaction requested=false
2024-12-11T20:13:34,615 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1588230740:

2024-12-11T20:13:34,620 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-12-11T20:13:34,620 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-11T20:13:34,620 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-11T20:13:34,620 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T20:13:34,621 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T20:13:34,621 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-12-11T20:13:34,621 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-11T20:13:34,621 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1232907217, stopped=false
2024-12-11T20:13:34,621 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=9fe0640122ec,40715,1733947988255
2024-12-11T20:13:34,622 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40715-0x100caf8ab8e0000, quorum=127.0.0.1:58530, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-11T20:13:34,623 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45215-0x100caf8ab8e0001, quorum=127.0.0.1:58530, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-11T20:13:34,623 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40715-0x100caf8ab8e0000, quorum=127.0.0.1:58530, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:13:34,623 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45215-0x100caf8ab8e0001, quorum=127.0.0.1:58530, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:13:34,623 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-11T20:13:34,623 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-11T20:13:34,623 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-11T20:13:34,623 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T20:13:34,623 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9fe0640122ec,45215,1733947988303' *****
2024-12-11T20:13:34,623 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-12-11T20:13:34,623 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40715-0x100caf8ab8e0000, quorum=127.0.0.1:58530, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-11T20:13:34,624 INFO  [RS:0;9fe0640122ec:45215 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-11T20:13:34,624 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45215-0x100caf8ab8e0001, quorum=127.0.0.1:58530, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-11T20:13:34,624 INFO  [RS:0;9fe0640122ec:45215 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-11T20:13:34,624 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-12-11T20:13:34,624 INFO  [RS:0;9fe0640122ec:45215 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-11T20:13:34,624 INFO  [RS:0;9fe0640122ec:45215 {}] regionserver.HRegionServer(3091): Received CLOSE for bf193a2f11156da1d1b7407be96c4423
2024-12-11T20:13:34,624 INFO  [RS:0;9fe0640122ec:45215 {}] regionserver.HRegionServer(959): stopping server 9fe0640122ec,45215,1733947988303
2024-12-11T20:13:34,624 INFO  [RS:0;9fe0640122ec:45215 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-11T20:13:34,624 INFO  [RS:0;9fe0640122ec:45215 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;9fe0640122ec:45215.
2024-12-11T20:13:34,624 DEBUG [RS:0;9fe0640122ec:45215 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-11T20:13:34,625 DEBUG [RS:0;9fe0640122ec:45215 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T20:13:34,625 INFO  [RS:0;9fe0640122ec:45215 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-11T20:13:34,625 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing bf193a2f11156da1d1b7407be96c4423, disabling compactions & flushes
2024-12-11T20:13:34,625 INFO  [RS:0;9fe0640122ec:45215 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-11T20:13:34,625 INFO  [RS:0;9fe0640122ec:45215 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-11T20:13:34,625 INFO  [RS:0;9fe0640122ec:45215 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-12-11T20:13:34,625 INFO  [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1733947989660.bf193a2f11156da1d1b7407be96c4423.
2024-12-11T20:13:34,625 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1733947989660.bf193a2f11156da1d1b7407be96c4423.
2024-12-11T20:13:34,625 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1733947989660.bf193a2f11156da1d1b7407be96c4423. after waiting 0 ms
2024-12-11T20:13:34,625 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1733947989660.bf193a2f11156da1d1b7407be96c4423.
2024-12-11T20:13:34,625 INFO  [RS:0;9fe0640122ec:45215 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-12-11T20:13:34,625 DEBUG [RS:0;9fe0640122ec:45215 {}] regionserver.HRegionServer(1325): Online Regions={bf193a2f11156da1d1b7407be96c4423=TestLogRolling-testLogRollOnPipelineRestart,,1733947989660.bf193a2f11156da1d1b7407be96c4423., 1588230740=hbase:meta,,1.1588230740}
2024-12-11T20:13:34,625 DEBUG [RS:0;9fe0640122ec:45215 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, bf193a2f11156da1d1b7407be96c4423
2024-12-11T20:13:34,625 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-11T20:13:34,625 INFO  [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-11T20:13:34,625 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-11T20:13:34,625 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-11T20:13:34,625 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-11T20:13:34,629 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/data/default/TestLogRolling-testLogRollOnPipelineRestart/bf193a2f11156da1d1b7407be96c4423/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1
2024-12-11T20:13:34,630 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-12-11T20:13:34,630 INFO  [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnPipelineRestart,,1733947989660.bf193a2f11156da1d1b7407be96c4423.
2024-12-11T20:13:34,630 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for bf193a2f11156da1d1b7407be96c4423:
Waiting for close lock at 1733948014624Running coprocessor pre-close hooks at 1733948014624Disabling compacts and flushes for region at 1733948014625 (+1 ms)Disabling writes for close at 1733948014625Writing region close event to WAL at 1733948014626 (+1 ms)Running coprocessor post-close hooks at 1733948014630 (+4 ms)Closed at 1733948014630
2024-12-11T20:13:34,630 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnPipelineRestart,,1733947989660.bf193a2f11156da1d1b7407be96c4423.
2024-12-11T20:13:34,631 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-11T20:13:34,631 INFO  [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-11T20:13:34,631 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733948014625Running coprocessor pre-close hooks at 1733948014625Disabling compacts and flushes for region at 1733948014625Disabling writes for close at 1733948014625Writing region close event to WAL at 1733948014627 (+2 ms)Running coprocessor post-close hooks at 1733948014631 (+4 ms)Closed at 1733948014631
2024-12-11T20:13:34,631 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-11T20:13:34,825 INFO  [RS:0;9fe0640122ec:45215 {}] regionserver.HRegionServer(976): stopping server 9fe0640122ec,45215,1733947988303; all regions closed.
2024-12-11T20:13:34,826 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:34,826 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:34,826 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:34,826 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:34,826 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:34,828 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46799 is added to blk_1073741842_1025 (size=825)
2024-12-11T20:13:34,828 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40081 is added to blk_1073741842_1025 (size=825)
2024-12-11T20:13:34,965 INFO  [regionserver/9fe0640122ec:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-12-11T20:13:34,965 INFO  [regionserver/9fe0640122ec:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-12-11T20:13:34,966 INFO  [regionserver/9fe0640122ec:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-11T20:13:35,283 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:35,284 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:36,283 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:36,284 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:37,284 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:37,285 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:38,237 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-11T20:13:38,285 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:38,285 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:38,490 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741834_1015: GenerationStamp not matched, existing replica is blk_1073741834_1010
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-12-11T20:13:38,520 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.meta.1733947989516.meta after 4001ms
2024-12-11T20:13:38,520 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/WALs/9fe0640122ec,45215,1733947988303/9fe0640122ec%2C45215%2C1733947988303.meta.1733947989516.meta to hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/oldWALs/9fe0640122ec%2C45215%2C1733947988303.meta.1733947989516.meta
2024-12-11T20:13:38,523 DEBUG [RS:0;9fe0640122ec:45215 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/oldWALs
2024-12-11T20:13:38,523 INFO  [RS:0;9fe0640122ec:45215 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 9fe0640122ec%2C45215%2C1733947988303.meta:.meta(num 1733948014512)
2024-12-11T20:13:38,524 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:38,524 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:38,524 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:38,524 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:38,524 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:38,526 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40081 is added to blk_1073741840_1023 (size=1162)
2024-12-11T20:13:38,527 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46799 is added to blk_1073741840_1023 (size=1162)
2024-12-11T20:13:38,531 DEBUG [RS:0;9fe0640122ec:45215 {}] wal.AbstractFSWAL(1256): Moved 4 WAL file(s) to /user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/oldWALs
2024-12-11T20:13:38,531 INFO  [RS:0;9fe0640122ec:45215 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 9fe0640122ec%2C45215%2C1733947988303:(num 1733948014466)
2024-12-11T20:13:38,531 DEBUG [RS:0;9fe0640122ec:45215 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T20:13:38,531 INFO  [RS:0;9fe0640122ec:45215 {}] regionserver.LeaseManager(133): Closed leases
2024-12-11T20:13:38,531 INFO  [RS:0;9fe0640122ec:45215 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-11T20:13:38,531 INFO  [RS:0;9fe0640122ec:45215 {}] hbase.ChoreService(370): Chore service for: regionserver/9fe0640122ec:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-12-11T20:13:38,531 INFO  [RS:0;9fe0640122ec:45215 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-11T20:13:38,531 INFO  [regionserver/9fe0640122ec:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-11T20:13:38,531 INFO  [RS:0;9fe0640122ec:45215 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45215
2024-12-11T20:13:38,533 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45215-0x100caf8ab8e0001, quorum=127.0.0.1:58530, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9fe0640122ec,45215,1733947988303
2024-12-11T20:13:38,533 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40715-0x100caf8ab8e0000, quorum=127.0.0.1:58530, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-11T20:13:38,533 INFO  [RS:0;9fe0640122ec:45215 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-11T20:13:38,535 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9fe0640122ec,45215,1733947988303]
2024-12-11T20:13:38,536 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9fe0640122ec,45215,1733947988303 already deleted, retry=false
2024-12-11T20:13:38,536 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 9fe0640122ec,45215,1733947988303 expired; onlineServers=0
2024-12-11T20:13:38,536 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '9fe0640122ec,40715,1733947988255' *****
2024-12-11T20:13:38,536 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-11T20:13:38,536 INFO  [M:0;9fe0640122ec:40715 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-11T20:13:38,536 INFO  [M:0;9fe0640122ec:40715 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-11T20:13:38,536 DEBUG [M:0;9fe0640122ec:40715 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-11T20:13:38,537 DEBUG [M:0;9fe0640122ec:40715 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-11T20:13:38,537 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-11T20:13:38,537 DEBUG [master/9fe0640122ec:0:becomeActiveMaster-HFileCleaner.small.0-1733947988914 {}] cleaner.HFileCleaner(306): Exit Thread[master/9fe0640122ec:0:becomeActiveMaster-HFileCleaner.small.0-1733947988914,5,FailOnTimeoutGroup]
2024-12-11T20:13:38,537 DEBUG [master/9fe0640122ec:0:becomeActiveMaster-HFileCleaner.large.0-1733947988914 {}] cleaner.HFileCleaner(306): Exit Thread[master/9fe0640122ec:0:becomeActiveMaster-HFileCleaner.large.0-1733947988914,5,FailOnTimeoutGroup]
2024-12-11T20:13:38,537 INFO  [M:0;9fe0640122ec:40715 {}] hbase.ChoreService(370): Chore service for: master/9fe0640122ec:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-12-11T20:13:38,537 INFO  [M:0;9fe0640122ec:40715 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-11T20:13:38,537 DEBUG [M:0;9fe0640122ec:40715 {}] master.HMaster(1795): Stopping service threads
2024-12-11T20:13:38,537 INFO  [M:0;9fe0640122ec:40715 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-11T20:13:38,537 INFO  [M:0;9fe0640122ec:40715 {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-11T20:13:38,537 INFO  [M:0;9fe0640122ec:40715 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-11T20:13:38,537 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-11T20:13:38,538 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40715-0x100caf8ab8e0000, quorum=127.0.0.1:58530, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-11T20:13:38,538 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40715-0x100caf8ab8e0000, quorum=127.0.0.1:58530, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:13:38,538 DEBUG [M:0;9fe0640122ec:40715 {}] zookeeper.ZKUtil(347): master:40715-0x100caf8ab8e0000, quorum=127.0.0.1:58530, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-11T20:13:38,538 WARN  [M:0;9fe0640122ec:40715 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-11T20:13:38,539 INFO  [M:0;9fe0640122ec:40715 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/.lastflushedseqids
2024-12-11T20:13:38,544 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40081 is added to blk_1073741846_1030 (size=130)
2024-12-11T20:13:38,545 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46799 is added to blk_1073741846_1030 (size=130)
2024-12-11T20:13:38,545 INFO  [M:0;9fe0640122ec:40715 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-12-11T20:13:38,545 INFO  [M:0;9fe0640122ec:40715 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-11T20:13:38,545 DEBUG [M:0;9fe0640122ec:40715 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-11T20:13:38,545 INFO  [M:0;9fe0640122ec:40715 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T20:13:38,545 DEBUG [M:0;9fe0640122ec:40715 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T20:13:38,545 DEBUG [M:0;9fe0640122ec:40715 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-11T20:13:38,545 DEBUG [M:0;9fe0640122ec:40715 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T20:13:38,545 INFO  [M:0;9fe0640122ec:40715 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.18 KB heapSize=29.16 KB
2024-12-11T20:13:38,546 ERROR [FSHLog-0-hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/MasterData-prefix:9fe0640122ec,40715,1733947988255 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33075,DS-7c862880-c949-4d2b-b189-775d30f1b8af,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:13:38,546 WARN  [FSHLog-0-hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/MasterData-prefix:9fe0640122ec,40715,1733947988255 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33075,DS-7c862880-c949-4d2b-b189-775d30f1b8af,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:13:38,546 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL FSHLog 9fe0640122ec%2C40715%2C1733947988255:(num 1733947988844) roll requested
2024-12-11T20:13:38,546 INFO  [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9fe0640122ec%2C40715%2C1733947988255.1733948018546
2024-12-11T20:13:38,551 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:38,551 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:38,551 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:38,551 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:38,551 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:38,551 INFO  [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/MasterData/WALs/9fe0640122ec,40715,1733947988255/9fe0640122ec%2C40715%2C1733947988255.1733947988844 with entries=53, filesize=26.63 KB; new WAL /user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/MasterData/WALs/9fe0640122ec,40715,1733947988255/9fe0640122ec%2C40715%2C1733947988255.1733948018546
2024-12-11T20:13:38,552 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33075,DS-7c862880-c949-4d2b-b189-775d30f1b8af,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:13:38,552 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33075,DS-7c862880-c949-4d2b-b189-775d30f1b8af,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-11T20:13:38,552 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/MasterData/WALs/9fe0640122ec,40715,1733947988255/9fe0640122ec%2C40715%2C1733947988255.1733947988844
2024-12-11T20:13:38,552 WARN  [IPC Server handler 1 on default port 42243 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/MasterData/WALs/9fe0640122ec,40715,1733947988255/9fe0640122ec%2C40715%2C1733947988255.1733947988844 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741830_1013
2024-12-11T20:13:38,552 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43617:43617),(127.0.0.1/127.0.0.1:36151:36151)]
2024-12-11T20:13:38,552 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/MasterData/WALs/9fe0640122ec,40715,1733947988255/9fe0640122ec%2C40715%2C1733947988255.1733947988844 is not closed yet, will try archiving it next time
2024-12-11T20:13:38,553 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/MasterData/WALs/9fe0640122ec,40715,1733947988255/9fe0640122ec%2C40715%2C1733947988255.1733947988844 after 1ms
2024-12-11T20:13:38,568 DEBUG [M:0;9fe0640122ec:40715 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4380e98000174c2887490d754dc09fa4 is 82, key is hbase:meta,,1/info:regioninfo/1733947989547/Put/seqid=0
2024-12-11T20:13:38,573 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40081 is added to blk_1073741848_1033 (size=5672)
2024-12-11T20:13:38,573 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46799 is added to blk_1073741848_1033 (size=5672)
2024-12-11T20:13:38,574 INFO  [M:0;9fe0640122ec:40715 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4380e98000174c2887490d754dc09fa4
2024-12-11T20:13:38,594 DEBUG [M:0;9fe0640122ec:40715 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/30c89536cdbb4702bd189a1857d7c6b1 is 779, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733947990052/Put/seqid=0
2024-12-11T20:13:38,599 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46799 is added to blk_1073741849_1034 (size=6119)
2024-12-11T20:13:38,600 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40081 is added to blk_1073741849_1034 (size=6119)
2024-12-11T20:13:38,600 INFO  [M:0;9fe0640122ec:40715 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.58 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/30c89536cdbb4702bd189a1857d7c6b1
2024-12-11T20:13:38,619 DEBUG [M:0;9fe0640122ec:40715 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e83ddeabb99a46899402633f35e49299 is 69, key is 9fe0640122ec,45215,1733947988303/rs:state/1733947988952/Put/seqid=0
2024-12-11T20:13:38,623 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46799 is added to blk_1073741850_1035 (size=5156)
2024-12-11T20:13:38,624 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40081 is added to blk_1073741850_1035 (size=5156)
2024-12-11T20:13:38,624 INFO  [M:0;9fe0640122ec:40715 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e83ddeabb99a46899402633f35e49299
2024-12-11T20:13:38,635 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45215-0x100caf8ab8e0001, quorum=127.0.0.1:58530, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-11T20:13:38,635 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45215-0x100caf8ab8e0001, quorum=127.0.0.1:58530, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-11T20:13:38,635 INFO  [RS:0;9fe0640122ec:45215 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-11T20:13:38,635 INFO  [RS:0;9fe0640122ec:45215 {}] regionserver.HRegionServer(1031): Exiting; stopping=9fe0640122ec,45215,1733947988303; zookeeper connection closed.
2024-12-11T20:13:38,635 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7b1b7510 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7b1b7510
2024-12-11T20:13:38,635 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-12-11T20:13:38,644 DEBUG [M:0;9fe0640122ec:40715 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/5d2f9035bd18414ca8a8ccda50ff6143 is 52, key is load_balancer_on/state:d/1733947989651/Put/seqid=0
2024-12-11T20:13:38,648 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40081 is added to blk_1073741851_1036 (size=5056)
2024-12-11T20:13:38,649 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46799 is added to blk_1073741851_1036 (size=5056)
2024-12-11T20:13:38,649 INFO  [M:0;9fe0640122ec:40715 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/5d2f9035bd18414ca8a8ccda50ff6143
2024-12-11T20:13:38,654 DEBUG [M:0;9fe0640122ec:40715 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4380e98000174c2887490d754dc09fa4 as hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/4380e98000174c2887490d754dc09fa4
2024-12-11T20:13:38,660 INFO  [M:0;9fe0640122ec:40715 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/4380e98000174c2887490d754dc09fa4, entries=8, sequenceid=56, filesize=5.5 K
2024-12-11T20:13:38,661 DEBUG [M:0;9fe0640122ec:40715 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/30c89536cdbb4702bd189a1857d7c6b1 as hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/30c89536cdbb4702bd189a1857d7c6b1
2024-12-11T20:13:38,665 INFO  [M:0;9fe0640122ec:40715 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/30c89536cdbb4702bd189a1857d7c6b1, entries=6, sequenceid=56, filesize=6.0 K
2024-12-11T20:13:38,666 DEBUG [M:0;9fe0640122ec:40715 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e83ddeabb99a46899402633f35e49299 as hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e83ddeabb99a46899402633f35e49299
2024-12-11T20:13:38,671 INFO  [M:0;9fe0640122ec:40715 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e83ddeabb99a46899402633f35e49299, entries=1, sequenceid=56, filesize=5.0 K
2024-12-11T20:13:38,672 DEBUG [M:0;9fe0640122ec:40715 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/5d2f9035bd18414ca8a8ccda50ff6143 as hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/5d2f9035bd18414ca8a8ccda50ff6143
2024-12-11T20:13:38,676 INFO  [M:0;9fe0640122ec:40715 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/5d2f9035bd18414ca8a8ccda50ff6143, entries=1, sequenceid=56, filesize=4.9 K
2024-12-11T20:13:38,677 INFO  [M:0;9fe0640122ec:40715 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.18 KB/23738, heapSize ~29.10 KB/29800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 132ms, sequenceid=56, compaction requested=false
2024-12-11T20:13:38,679 INFO  [M:0;9fe0640122ec:40715 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T20:13:38,679 DEBUG [M:0;9fe0640122ec:40715 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733948018545Disabling compacts and flushes for region at 1733948018545Disabling writes for close at 1733948018545Obtaining lock to block concurrent updates at 1733948018545Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733948018545Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23738, getHeapSize=29800, getOffHeapSize=0, getCellsCount=67 at 1733948018546 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733948018553 (+7 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733948018553Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733948018567 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733948018567Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733948018579 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733948018594 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733948018594Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733948018604 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733948018618 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733948018618Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733948018629 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733948018643 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733948018643Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3bbf12f7: reopening flushed file at 1733948018654 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@772865df: reopening flushed file at 1733948018660 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7f7d75f0: reopening flushed file at 1733948018665 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4c972b96: reopening flushed file at 1733948018671 (+6 ms)Finished flush of dataSize ~23.18 KB/23738, heapSize ~29.10 KB/29800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 132ms, sequenceid=56, compaction requested=false at 1733948018677 (+6 ms)Writing region close event to WAL at 1733948018679 (+2 ms)Closed at 1733948018679
2024-12-11T20:13:38,679 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:38,679 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:38,679 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:38,679 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:38,680 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:13:38,681 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40081 is added to blk_1073741847_1031 (size=757)
2024-12-11T20:13:38,681 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46799 is added to blk_1073741847_1031 (size=757)
2024-12-11T20:13:39,285 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:39,285 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:39,631 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:39,631 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:39,642 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:39,643 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:39,643 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:39,643 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:39,643 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:39,643 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:39,646 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:39,646 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:39,646 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:39,648 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:39,651 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:39,652 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:40,154 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-11T20:13:40,155 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:40,156 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:40,156 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:40,156 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:40,170 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:40,170 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:40,170 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:40,170 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:40,171 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:40,171 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:40,174 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:40,174 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:40,174 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:40,177 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:40,286 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:40,286 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:41,286 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:41,286 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:41,490 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741830_1013: GenerationStamp not matched, existing replica is blk_1073741830_1006
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-12-11T20:13:42,156 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-11T20:13:42,156 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-12-11T20:13:42,156 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-12-11T20:13:42,156 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart
2024-12-11T20:13:42,287 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:42,287 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:42,553 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/MasterData/WALs/9fe0640122ec,40715,1733947988255/9fe0640122ec%2C40715%2C1733947988255.1733947988844 after 4001ms
2024-12-11T20:13:42,554 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/MasterData/WALs/9fe0640122ec,40715,1733947988255/9fe0640122ec%2C40715%2C1733947988255.1733947988844 to hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/MasterData/oldWALs/9fe0640122ec%2C40715%2C1733947988255.1733947988844
2024-12-11T20:13:42,557 INFO  [WAL-Archive-0 {}] region.MasterRegionUtils(50): Moved hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/MasterData/oldWALs/9fe0640122ec%2C40715%2C1733947988255.1733947988844 to hdfs://localhost:42243/user/jenkins/test-data/c4a3014a-9e9d-7cf1-43ed-f07ecfb63fa3/oldWALs/9fe0640122ec%2C40715%2C1733947988255.1733947988844$masterlocalwal$
2024-12-11T20:13:42,557 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-11T20:13:42,557 INFO  [M:0;9fe0640122ec:40715 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-12-11T20:13:42,557 INFO  [M:0;9fe0640122ec:40715 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40715
2024-12-11T20:13:42,557 INFO  [M:0;9fe0640122ec:40715 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-11T20:13:42,660 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40715-0x100caf8ab8e0000, quorum=127.0.0.1:58530, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-11T20:13:42,660 INFO  [M:0;9fe0640122ec:40715 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-11T20:13:42,660 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40715-0x100caf8ab8e0000, quorum=127.0.0.1:58530, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-11T20:13:42,662 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@77ebbd9c{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T20:13:42,663 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@14b40a8a{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-11T20:13:42,663 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-11T20:13:42,663 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@bc30fa1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-11T20:13:42,663 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4f7ae33f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6761f64-4d8d-e744-edc5-9c12798c8e6a/hadoop.log.dir/,STOPPED}
2024-12-11T20:13:42,664 WARN  [BP-558819663-172.17.0.2-1733947987520 heartbeating to localhost/127.0.0.1:42243 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-11T20:13:42,664 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-11T20:13:42,664 WARN  [BP-558819663-172.17.0.2-1733947987520 heartbeating to localhost/127.0.0.1:42243 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-558819663-172.17.0.2-1733947987520 (Datanode Uuid c24dc4a7-9b08-451a-82ef-036733e0d594) service to localhost/127.0.0.1:42243
2024-12-11T20:13:42,664 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-11T20:13:42,665 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6761f64-4d8d-e744-edc5-9c12798c8e6a/cluster_e6efd374-2de3-88d0-f464-6076ba3d414f/data/data3/current/BP-558819663-172.17.0.2-1733947987520 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T20:13:42,665 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6761f64-4d8d-e744-edc5-9c12798c8e6a/cluster_e6efd374-2de3-88d0-f464-6076ba3d414f/data/data4/current/BP-558819663-172.17.0.2-1733947987520 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T20:13:42,665 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-11T20:13:42,669 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2026736d{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T20:13:42,670 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@22b034d5{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-11T20:13:42,670 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-11T20:13:42,670 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@731665df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-11T20:13:42,670 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@78cc0925{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6761f64-4d8d-e744-edc5-9c12798c8e6a/hadoop.log.dir/,STOPPED}
2024-12-11T20:13:42,671 WARN  [BP-558819663-172.17.0.2-1733947987520 heartbeating to localhost/127.0.0.1:42243 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-11T20:13:42,671 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-11T20:13:42,671 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-11T20:13:42,671 WARN  [BP-558819663-172.17.0.2-1733947987520 heartbeating to localhost/127.0.0.1:42243 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-558819663-172.17.0.2-1733947987520 (Datanode Uuid 4c3e95e7-fe9c-4d12-ba2c-9bb9508f688f) service to localhost/127.0.0.1:42243
2024-12-11T20:13:42,672 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6761f64-4d8d-e744-edc5-9c12798c8e6a/cluster_e6efd374-2de3-88d0-f464-6076ba3d414f/data/data1/current/BP-558819663-172.17.0.2-1733947987520 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T20:13:42,672 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6761f64-4d8d-e744-edc5-9c12798c8e6a/cluster_e6efd374-2de3-88d0-f464-6076ba3d414f/data/data2/current/BP-558819663-172.17.0.2-1733947987520 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T20:13:42,672 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-11T20:13:42,677 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@533cae40{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-11T20:13:42,678 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1e98c3cf{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-11T20:13:42,678 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-11T20:13:42,678 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1a59d25d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-11T20:13:42,678 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5e4fac25{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6761f64-4d8d-e744-edc5-9c12798c8e6a/hadoop.log.dir/,STOPPED}
2024-12-11T20:13:42,684 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-12-11T20:13:42,701 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-12-11T20:13:42,709 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=179 (was 162)
Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-33-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-32-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:42243
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-32-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:42243 from jenkins.hfs.4
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: HMaster-EventLoopGroup-10-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.4@localhost:42243
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-30-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-31-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-32-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:42243
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-31-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-30-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-33-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-10-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:42243 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:42243
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-31-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:42243
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-33-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-10-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-30-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:42243 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)
 - Thread LEAK? -, OpenFileDescriptor=457 (was 450) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=138 (was 207), ProcessCount=11 (was 11), AvailableMemoryMB=293 (was 343)
2024-12-11T20:13:42,716 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=179, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=138, ProcessCount=11, AvailableMemoryMB=293
2024-12-11T20:13:42,717 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-11T20:13:42,717 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6761f64-4d8d-e744-edc5-9c12798c8e6a/hadoop.log.dir so I do NOT create it in target/test-data/a9019bd4-2e2e-eb94-0662-008269ec518f
2024-12-11T20:13:42,717 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6761f64-4d8d-e744-edc5-9c12798c8e6a/hadoop.tmp.dir so I do NOT create it in target/test-data/a9019bd4-2e2e-eb94-0662-008269ec518f
2024-12-11T20:13:42,717 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a9019bd4-2e2e-eb94-0662-008269ec518f/cluster_e92aeb1d-de81-341f-1b6e-20867c1a221d, deleteOnExit=true
2024-12-11T20:13:42,717 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-12-11T20:13:42,717 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a9019bd4-2e2e-eb94-0662-008269ec518f/test.cache.data in system properties and HBase conf
2024-12-11T20:13:42,717 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a9019bd4-2e2e-eb94-0662-008269ec518f/hadoop.tmp.dir in system properties and HBase conf
2024-12-11T20:13:42,717 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a9019bd4-2e2e-eb94-0662-008269ec518f/hadoop.log.dir in system properties and HBase conf
2024-12-11T20:13:42,717 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a9019bd4-2e2e-eb94-0662-008269ec518f/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-11T20:13:42,717 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a9019bd4-2e2e-eb94-0662-008269ec518f/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-11T20:13:42,717 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-12-11T20:13:42,717 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-11T20:13:42,718 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a9019bd4-2e2e-eb94-0662-008269ec518f/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-11T20:13:42,718 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a9019bd4-2e2e-eb94-0662-008269ec518f/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-11T20:13:42,718 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a9019bd4-2e2e-eb94-0662-008269ec518f/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-11T20:13:42,718 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a9019bd4-2e2e-eb94-0662-008269ec518f/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-11T20:13:42,718 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a9019bd4-2e2e-eb94-0662-008269ec518f/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-11T20:13:42,718 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a9019bd4-2e2e-eb94-0662-008269ec518f/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-11T20:13:42,718 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a9019bd4-2e2e-eb94-0662-008269ec518f/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-11T20:13:42,718 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a9019bd4-2e2e-eb94-0662-008269ec518f/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-11T20:13:42,718 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a9019bd4-2e2e-eb94-0662-008269ec518f/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-11T20:13:42,718 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a9019bd4-2e2e-eb94-0662-008269ec518f/nfs.dump.dir in system properties and HBase conf
2024-12-11T20:13:42,718 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a9019bd4-2e2e-eb94-0662-008269ec518f/java.io.tmpdir in system properties and HBase conf
2024-12-11T20:13:42,718 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a9019bd4-2e2e-eb94-0662-008269ec518f/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-11T20:13:42,718 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a9019bd4-2e2e-eb94-0662-008269ec518f/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-11T20:13:42,718 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a9019bd4-2e2e-eb94-0662-008269ec518f/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-11T20:13:42,731 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-11T20:13:42,796 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T20:13:42,801 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-11T20:13:42,805 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-11T20:13:42,805 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-11T20:13:42,805 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-11T20:13:42,805 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T20:13:42,806 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@c464107{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a9019bd4-2e2e-eb94-0662-008269ec518f/hadoop.log.dir/,AVAILABLE}
2024-12-11T20:13:42,806 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@29b40997{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-11T20:13:42,921 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@21edc7fd{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a9019bd4-2e2e-eb94-0662-008269ec518f/java.io.tmpdir/jetty-localhost-43899-hadoop-hdfs-3_4_1-tests_jar-_-any-3979494276813486681/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-11T20:13:42,922 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7b612817{HTTP/1.1, (http/1.1)}{localhost:43899}
2024-12-11T20:13:42,922 INFO  [Time-limited test {}] server.Server(415): Started @186518ms
2024-12-11T20:13:42,936 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-11T20:13:42,983 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T20:13:42,986 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-11T20:13:42,986 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-11T20:13:42,986 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-11T20:13:42,986 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-11T20:13:42,987 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@b24fbcd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a9019bd4-2e2e-eb94-0662-008269ec518f/hadoop.log.dir/,AVAILABLE}
2024-12-11T20:13:42,987 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@16aeea80{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-11T20:13:43,103 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5d8d7f9b{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a9019bd4-2e2e-eb94-0662-008269ec518f/java.io.tmpdir/jetty-localhost-41351-hadoop-hdfs-3_4_1-tests_jar-_-any-6315860500095368807/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T20:13:43,104 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@190ad9e7{HTTP/1.1, (http/1.1)}{localhost:41351}
2024-12-11T20:13:43,104 INFO  [Time-limited test {}] server.Server(415): Started @186699ms
2024-12-11T20:13:43,105 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-11T20:13:43,133 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T20:13:43,136 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-11T20:13:43,136 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-11T20:13:43,136 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-11T20:13:43,137 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-11T20:13:43,137 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@72ef9fa2{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a9019bd4-2e2e-eb94-0662-008269ec518f/hadoop.log.dir/,AVAILABLE}
2024-12-11T20:13:43,137 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@438a440e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-11T20:13:43,184 WARN  [Thread-1642 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a9019bd4-2e2e-eb94-0662-008269ec518f/cluster_e92aeb1d-de81-341f-1b6e-20867c1a221d/data/data1/current/BP-313382295-172.17.0.2-1733948022749/current, will proceed with Du for space computation calculation, 
2024-12-11T20:13:43,184 WARN  [Thread-1643 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a9019bd4-2e2e-eb94-0662-008269ec518f/cluster_e92aeb1d-de81-341f-1b6e-20867c1a221d/data/data2/current/BP-313382295-172.17.0.2-1733948022749/current, will proceed with Du for space computation calculation, 
2024-12-11T20:13:43,205 WARN  [Thread-1621 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-11T20:13:43,208 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3599287a7fe0c32 with lease ID 0x1b3dee5bb0feec1a: Processing first storage report for DS-50aadfac-4f7f-441c-825a-9d4207b5bb29 from datanode DatanodeRegistration(127.0.0.1:43661, datanodeUuid=1d9df031-89dd-472d-b7cd-4182e4049549, infoPort=44059, infoSecurePort=0, ipcPort=42405, storageInfo=lv=-57;cid=testClusterID;nsid=1963926947;c=1733948022749)
2024-12-11T20:13:43,208 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3599287a7fe0c32 with lease ID 0x1b3dee5bb0feec1a: from storage DS-50aadfac-4f7f-441c-825a-9d4207b5bb29 node DatanodeRegistration(127.0.0.1:43661, datanodeUuid=1d9df031-89dd-472d-b7cd-4182e4049549, infoPort=44059, infoSecurePort=0, ipcPort=42405, storageInfo=lv=-57;cid=testClusterID;nsid=1963926947;c=1733948022749), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-11T20:13:43,208 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3599287a7fe0c32 with lease ID 0x1b3dee5bb0feec1a: Processing first storage report for DS-4c5836b8-295e-46a8-b9b3-99fb57ed781e from datanode DatanodeRegistration(127.0.0.1:43661, datanodeUuid=1d9df031-89dd-472d-b7cd-4182e4049549, infoPort=44059, infoSecurePort=0, ipcPort=42405, storageInfo=lv=-57;cid=testClusterID;nsid=1963926947;c=1733948022749)
2024-12-11T20:13:43,208 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3599287a7fe0c32 with lease ID 0x1b3dee5bb0feec1a: from storage DS-4c5836b8-295e-46a8-b9b3-99fb57ed781e node DatanodeRegistration(127.0.0.1:43661, datanodeUuid=1d9df031-89dd-472d-b7cd-4182e4049549, infoPort=44059, infoSecurePort=0, ipcPort=42405, storageInfo=lv=-57;cid=testClusterID;nsid=1963926947;c=1733948022749), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T20:13:43,258 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3d40a54d{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a9019bd4-2e2e-eb94-0662-008269ec518f/java.io.tmpdir/jetty-localhost-39567-hadoop-hdfs-3_4_1-tests_jar-_-any-4950005221336498/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T20:13:43,259 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@74b5ebca{HTTP/1.1, (http/1.1)}{localhost:39567}
2024-12-11T20:13:43,259 INFO  [Time-limited test {}] server.Server(415): Started @186854ms
2024-12-11T20:13:43,260 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-11T20:13:43,288 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:43,288 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:43,338 WARN  [Thread-1668 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a9019bd4-2e2e-eb94-0662-008269ec518f/cluster_e92aeb1d-de81-341f-1b6e-20867c1a221d/data/data3/current/BP-313382295-172.17.0.2-1733948022749/current, will proceed with Du for space computation calculation, 
2024-12-11T20:13:43,338 WARN  [Thread-1669 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a9019bd4-2e2e-eb94-0662-008269ec518f/cluster_e92aeb1d-de81-341f-1b6e-20867c1a221d/data/data4/current/BP-313382295-172.17.0.2-1733948022749/current, will proceed with Du for space computation calculation, 
2024-12-11T20:13:43,354 WARN  [Thread-1657 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-11T20:13:43,356 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x85168f0d079b99e4 with lease ID 0x1b3dee5bb0feec1b: Processing first storage report for DS-566a6b7d-5e7b-41f4-99d0-f762aca3b555 from datanode DatanodeRegistration(127.0.0.1:38159, datanodeUuid=56bb8b2c-8d4a-4065-a135-d006da89b1ef, infoPort=37643, infoSecurePort=0, ipcPort=33343, storageInfo=lv=-57;cid=testClusterID;nsid=1963926947;c=1733948022749)
2024-12-11T20:13:43,356 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x85168f0d079b99e4 with lease ID 0x1b3dee5bb0feec1b: from storage DS-566a6b7d-5e7b-41f4-99d0-f762aca3b555 node DatanodeRegistration(127.0.0.1:38159, datanodeUuid=56bb8b2c-8d4a-4065-a135-d006da89b1ef, infoPort=37643, infoSecurePort=0, ipcPort=33343, storageInfo=lv=-57;cid=testClusterID;nsid=1963926947;c=1733948022749), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T20:13:43,356 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x85168f0d079b99e4 with lease ID 0x1b3dee5bb0feec1b: Processing first storage report for DS-ed972bca-171b-461b-abf2-28164a205cb6 from datanode DatanodeRegistration(127.0.0.1:38159, datanodeUuid=56bb8b2c-8d4a-4065-a135-d006da89b1ef, infoPort=37643, infoSecurePort=0, ipcPort=33343, storageInfo=lv=-57;cid=testClusterID;nsid=1963926947;c=1733948022749)
2024-12-11T20:13:43,357 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x85168f0d079b99e4 with lease ID 0x1b3dee5bb0feec1b: from storage DS-ed972bca-171b-461b-abf2-28164a205cb6 node DatanodeRegistration(127.0.0.1:38159, datanodeUuid=56bb8b2c-8d4a-4065-a135-d006da89b1ef, infoPort=37643, infoSecurePort=0, ipcPort=33343, storageInfo=lv=-57;cid=testClusterID;nsid=1963926947;c=1733948022749), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T20:13:43,383 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a9019bd4-2e2e-eb94-0662-008269ec518f
2024-12-11T20:13:43,389 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a9019bd4-2e2e-eb94-0662-008269ec518f/cluster_e92aeb1d-de81-341f-1b6e-20867c1a221d/zookeeper_0, clientPort=63467, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a9019bd4-2e2e-eb94-0662-008269ec518f/cluster_e92aeb1d-de81-341f-1b6e-20867c1a221d/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a9019bd4-2e2e-eb94-0662-008269ec518f/cluster_e92aeb1d-de81-341f-1b6e-20867c1a221d/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-11T20:13:43,390 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=63467
2024-12-11T20:13:43,390 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T20:13:43,391 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T20:13:43,399 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43661 is added to blk_1073741825_1001 (size=7)
2024-12-11T20:13:43,400 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38159 is added to blk_1073741825_1001 (size=7)
2024-12-11T20:13:43,401 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b with version=8
2024-12-11T20:13:43,401 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/hbase-staging
2024-12-11T20:13:43,402 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/9fe0640122ec:0 server-side Connection retries=45
2024-12-11T20:13:43,403 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-11T20:13:43,403 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-11T20:13:43,403 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-11T20:13:43,403 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-11T20:13:43,403 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-11T20:13:43,403 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-12-11T20:13:43,403 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-11T20:13:43,404 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35237
2024-12-11T20:13:43,405 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35237 connecting to ZooKeeper ensemble=127.0.0.1:63467
2024-12-11T20:13:43,410 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:352370x0, quorum=127.0.0.1:63467, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-11T20:13:43,410 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35237-0x100caf934d80000 connected
2024-12-11T20:13:43,423 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T20:13:43,424 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T20:13:43,426 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35237-0x100caf934d80000, quorum=127.0.0.1:63467, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-11T20:13:43,426 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b, hbase.cluster.distributed=false
2024-12-11T20:13:43,427 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35237-0x100caf934d80000, quorum=127.0.0.1:63467, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-11T20:13:43,428 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35237
2024-12-11T20:13:43,428 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35237
2024-12-11T20:13:43,428 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35237
2024-12-11T20:13:43,429 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35237
2024-12-11T20:13:43,430 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35237
2024-12-11T20:13:43,445 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/9fe0640122ec:0 server-side Connection retries=45
2024-12-11T20:13:43,445 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-11T20:13:43,445 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-11T20:13:43,445 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-11T20:13:43,445 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-11T20:13:43,445 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-11T20:13:43,446 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-11T20:13:43,446 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-11T20:13:43,446 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34019
2024-12-11T20:13:43,448 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34019 connecting to ZooKeeper ensemble=127.0.0.1:63467
2024-12-11T20:13:43,448 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T20:13:43,450 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T20:13:43,454 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:340190x0, quorum=127.0.0.1:63467, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-11T20:13:43,454 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:340190x0, quorum=127.0.0.1:63467, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-11T20:13:43,454 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34019-0x100caf934d80001 connected
2024-12-11T20:13:43,454 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-11T20:13:43,455 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-11T20:13:43,456 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34019-0x100caf934d80001, quorum=127.0.0.1:63467, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-11T20:13:43,457 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34019-0x100caf934d80001, quorum=127.0.0.1:63467, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-11T20:13:43,457 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34019
2024-12-11T20:13:43,457 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34019
2024-12-11T20:13:43,460 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34019
2024-12-11T20:13:43,461 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34019
2024-12-11T20:13:43,462 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34019
2024-12-11T20:13:43,474 DEBUG [M:0;9fe0640122ec:35237 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;9fe0640122ec:35237
2024-12-11T20:13:43,474 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/9fe0640122ec,35237,1733948023402
2024-12-11T20:13:43,476 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35237-0x100caf934d80000, quorum=127.0.0.1:63467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-11T20:13:43,476 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34019-0x100caf934d80001, quorum=127.0.0.1:63467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-11T20:13:43,476 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35237-0x100caf934d80000, quorum=127.0.0.1:63467, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/9fe0640122ec,35237,1733948023402
2024-12-11T20:13:43,478 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34019-0x100caf934d80001, quorum=127.0.0.1:63467, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-11T20:13:43,478 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35237-0x100caf934d80000, quorum=127.0.0.1:63467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:13:43,478 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34019-0x100caf934d80001, quorum=127.0.0.1:63467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:13:43,478 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35237-0x100caf934d80000, quorum=127.0.0.1:63467, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-11T20:13:43,479 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/9fe0640122ec,35237,1733948023402 from backup master directory
2024-12-11T20:13:43,480 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35237-0x100caf934d80000, quorum=127.0.0.1:63467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/9fe0640122ec,35237,1733948023402
2024-12-11T20:13:43,480 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34019-0x100caf934d80001, quorum=127.0.0.1:63467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-11T20:13:43,480 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35237-0x100caf934d80000, quorum=127.0.0.1:63467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-11T20:13:43,480 WARN  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-11T20:13:43,480 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=9fe0640122ec,35237,1733948023402
2024-12-11T20:13:43,484 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/hbase.id] with ID: dc791c5c-573b-4a4c-a995-329f7178dd77
2024-12-11T20:13:43,484 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/.tmp/hbase.id
2024-12-11T20:13:43,490 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38159 is added to blk_1073741826_1002 (size=42)
2024-12-11T20:13:43,491 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43661 is added to blk_1073741826_1002 (size=42)
2024-12-11T20:13:43,491 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/.tmp/hbase.id]:[hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/hbase.id]
2024-12-11T20:13:43,502 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T20:13:43,502 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-12-11T20:13:43,503 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-12-11T20:13:43,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35237-0x100caf934d80000, quorum=127.0.0.1:63467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:13:43,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34019-0x100caf934d80001, quorum=127.0.0.1:63467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:13:43,512 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38159 is added to blk_1073741827_1003 (size=196)
2024-12-11T20:13:43,512 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43661 is added to blk_1073741827_1003 (size=196)
2024-12-11T20:13:43,513 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-11T20:13:43,514 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-11T20:13:43,514 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-11T20:13:43,521 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38159 is added to blk_1073741828_1004 (size=1189)
2024-12-11T20:13:43,522 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43661 is added to blk_1073741828_1004 (size=1189)
2024-12-11T20:13:43,522 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/MasterData/data/master/store
2024-12-11T20:13:43,529 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38159 is added to blk_1073741829_1005 (size=34)
2024-12-11T20:13:43,530 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43661 is added to blk_1073741829_1005 (size=34)
2024-12-11T20:13:43,530 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T20:13:43,530 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-11T20:13:43,530 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T20:13:43,530 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T20:13:43,530 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-11T20:13:43,530 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T20:13:43,530 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T20:13:43,530 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733948023530Disabling compacts and flushes for region at 1733948023530Disabling writes for close at 1733948023530Writing region close event to WAL at 1733948023530Closed at 1733948023530
2024-12-11T20:13:43,531 WARN  [master/9fe0640122ec:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/MasterData/data/master/store/.initializing
2024-12-11T20:13:43,531 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/MasterData/WALs/9fe0640122ec,35237,1733948023402
2024-12-11T20:13:43,534 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9fe0640122ec%2C35237%2C1733948023402, suffix=, logDir=hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/MasterData/WALs/9fe0640122ec,35237,1733948023402, archiveDir=hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/MasterData/oldWALs, maxLogs=10
2024-12-11T20:13:43,534 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9fe0640122ec%2C35237%2C1733948023402.1733948023534
2024-12-11T20:13:43,538 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/MasterData/WALs/9fe0640122ec,35237,1733948023402/9fe0640122ec%2C35237%2C1733948023402.1733948023534
2024-12-11T20:13:43,539 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44059:44059),(127.0.0.1/127.0.0.1:37643:37643)]
2024-12-11T20:13:43,540 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-11T20:13:43,540 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T20:13:43,540 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T20:13:43,540 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T20:13:43,541 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-11T20:13:43,543 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-11T20:13:43,543 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:13:43,543 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T20:13:43,543 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-11T20:13:43,544 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-11T20:13:43,544 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:13:43,545 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-11T20:13:43,545 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-11T20:13:43,545 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-11T20:13:43,546 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:13:43,546 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-11T20:13:43,546 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-11T20:13:43,547 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-11T20:13:43,547 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:13:43,547 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-11T20:13:43,547 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T20:13:43,548 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-11T20:13:43,548 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-11T20:13:43,550 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T20:13:43,550 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T20:13:43,551 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-11T20:13:43,552 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T20:13:43,554 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-11T20:13:43,554 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=850762, jitterRate=0.08180032670497894}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-11T20:13:43,555 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1733948023540Initializing all the Stores at 1733948023541 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733948023541Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733948023541Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733948023541Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733948023541Cleaning up temporary data from old regions at 1733948023550 (+9 ms)Region opened successfully at 1733948023554 (+4 ms)
2024-12-11T20:13:43,555 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-11T20:13:43,558 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41747c96, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9fe0640122ec/172.17.0.2:0
2024-12-11T20:13:43,559 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-12-11T20:13:43,559 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-11T20:13:43,559 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-11T20:13:43,559 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-11T20:13:43,559 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-12-11T20:13:43,560 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-12-11T20:13:43,560 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-11T20:13:43,561 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-11T20:13:43,562 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35237-0x100caf934d80000, quorum=127.0.0.1:63467, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-11T20:13:43,563 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-12-11T20:13:43,564 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-11T20:13:43,564 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35237-0x100caf934d80000, quorum=127.0.0.1:63467, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-11T20:13:43,566 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-12-11T20:13:43,566 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-11T20:13:43,567 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35237-0x100caf934d80000, quorum=127.0.0.1:63467, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-11T20:13:43,568 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-12-11T20:13:43,569 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35237-0x100caf934d80000, quorum=127.0.0.1:63467, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-11T20:13:43,570 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-12-11T20:13:43,571 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35237-0x100caf934d80000, quorum=127.0.0.1:63467, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-11T20:13:43,572 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-11T20:13:43,574 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35237-0x100caf934d80000, quorum=127.0.0.1:63467, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-11T20:13:43,574 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34019-0x100caf934d80001, quorum=127.0.0.1:63467, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-11T20:13:43,574 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35237-0x100caf934d80000, quorum=127.0.0.1:63467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:13:43,574 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34019-0x100caf934d80001, quorum=127.0.0.1:63467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:13:43,574 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=9fe0640122ec,35237,1733948023402, sessionid=0x100caf934d80000, setting cluster-up flag (Was=false)
2024-12-11T20:13:43,576 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34019-0x100caf934d80001, quorum=127.0.0.1:63467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:13:43,576 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35237-0x100caf934d80000, quorum=127.0.0.1:63467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:13:43,581 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-11T20:13:43,581 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9fe0640122ec,35237,1733948023402
2024-12-11T20:13:43,584 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35237-0x100caf934d80000, quorum=127.0.0.1:63467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:13:43,584 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34019-0x100caf934d80001, quorum=127.0.0.1:63467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:13:43,588 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-11T20:13:43,589 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9fe0640122ec,35237,1733948023402
2024-12-11T20:13:43,591 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-12-11T20:13:43,592 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-12-11T20:13:43,593 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-12-11T20:13:43,593 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-11T20:13:43,593 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 9fe0640122ec,35237,1733948023402
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-11T20:13:43,594 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/9fe0640122ec:0, corePoolSize=5, maxPoolSize=5
2024-12-11T20:13:43,594 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/9fe0640122ec:0, corePoolSize=5, maxPoolSize=5
2024-12-11T20:13:43,594 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/9fe0640122ec:0, corePoolSize=5, maxPoolSize=5
2024-12-11T20:13:43,594 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/9fe0640122ec:0, corePoolSize=5, maxPoolSize=5
2024-12-11T20:13:43,594 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/9fe0640122ec:0, corePoolSize=10, maxPoolSize=10
2024-12-11T20:13:43,594 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:13:43,594 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/9fe0640122ec:0, corePoolSize=2, maxPoolSize=2
2024-12-11T20:13:43,594 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:13:43,595 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733948053595
2024-12-11T20:13:43,595 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-11T20:13:43,595 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-11T20:13:43,595 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-11T20:13:43,595 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-11T20:13:43,596 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-11T20:13:43,596 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-11T20:13:43,596 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-11T20:13:43,596 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-11T20:13:43,596 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-11T20:13:43,596 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-11T20:13:43,596 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-12-11T20:13:43,596 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-11T20:13:43,597 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:13:43,597 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-11T20:13:43,600 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-11T20:13:43,600 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-11T20:13:43,600 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/9fe0640122ec:0:becomeActiveMaster-HFileCleaner.large.0-1733948023600,5,FailOnTimeoutGroup]
2024-12-11T20:13:43,600 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/9fe0640122ec:0:becomeActiveMaster-HFileCleaner.small.0-1733948023600,5,FailOnTimeoutGroup]
2024-12-11T20:13:43,600 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-11T20:13:43,600 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-11T20:13:43,600 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-11T20:13:43,600 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-11T20:13:43,605 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43661 is added to blk_1073741831_1007 (size=1321)
2024-12-11T20:13:43,605 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38159 is added to blk_1073741831_1007 (size=1321)
2024-12-11T20:13:43,606 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-12-11T20:13:43,606 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b
2024-12-11T20:13:43,612 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43661 is added to blk_1073741832_1008 (size=32)
2024-12-11T20:13:43,612 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38159 is added to blk_1073741832_1008 (size=32)
2024-12-11T20:13:43,612 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T20:13:43,613 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-11T20:13:43,615 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-11T20:13:43,615 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:13:43,615 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T20:13:43,615 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-11T20:13:43,616 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-11T20:13:43,616 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:13:43,617 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T20:13:43,617 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-11T20:13:43,618 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-11T20:13:43,618 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:13:43,618 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T20:13:43,618 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-11T20:13:43,619 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-11T20:13:43,619 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:13:43,619 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T20:13:43,620 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-11T20:13:43,620 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/data/hbase/meta/1588230740
2024-12-11T20:13:43,620 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/data/hbase/meta/1588230740
2024-12-11T20:13:43,622 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-11T20:13:43,622 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-11T20:13:43,622 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-11T20:13:43,623 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-11T20:13:43,625 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-11T20:13:43,625 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=805539, jitterRate=0.024295851588249207}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-11T20:13:43,626 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1733948023613Initializing all the Stores at 1733948023613Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733948023613Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733948023613Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733948023613Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733948023613Cleaning up temporary data from old regions at 1733948023622 (+9 ms)Region opened successfully at 1733948023626 (+4 ms)
2024-12-11T20:13:43,626 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-11T20:13:43,626 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-11T20:13:43,626 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-11T20:13:43,626 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-11T20:13:43,626 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-11T20:13:43,626 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-11T20:13:43,626 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733948023626Disabling compacts and flushes for region at 1733948023626Disabling writes for close at 1733948023626Writing region close event to WAL at 1733948023626Closed at 1733948023626
2024-12-11T20:13:43,627 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-11T20:13:43,628 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-12-11T20:13:43,628 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-11T20:13:43,629 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-11T20:13:43,630 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-11T20:13:43,664 INFO  [RS:0;9fe0640122ec:34019 {}] regionserver.HRegionServer(746): ClusterId : dc791c5c-573b-4a4c-a995-329f7178dd77
2024-12-11T20:13:43,664 DEBUG [RS:0;9fe0640122ec:34019 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-11T20:13:43,666 DEBUG [RS:0;9fe0640122ec:34019 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-11T20:13:43,666 DEBUG [RS:0;9fe0640122ec:34019 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-11T20:13:43,667 DEBUG [RS:0;9fe0640122ec:34019 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-11T20:13:43,668 DEBUG [RS:0;9fe0640122ec:34019 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d206b5a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9fe0640122ec/172.17.0.2:0
2024-12-11T20:13:43,680 DEBUG [RS:0;9fe0640122ec:34019 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;9fe0640122ec:34019
2024-12-11T20:13:43,680 INFO  [RS:0;9fe0640122ec:34019 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-11T20:13:43,680 INFO  [RS:0;9fe0640122ec:34019 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-11T20:13:43,680 DEBUG [RS:0;9fe0640122ec:34019 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-11T20:13:43,680 INFO  [RS:0;9fe0640122ec:34019 {}] regionserver.HRegionServer(2659): reportForDuty to master=9fe0640122ec,35237,1733948023402 with port=34019, startcode=1733948023445
2024-12-11T20:13:43,681 DEBUG [RS:0;9fe0640122ec:34019 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-11T20:13:43,683 INFO  [HMaster-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35243, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.5 (auth:SIMPLE), service=RegionServerStatusService
2024-12-11T20:13:43,683 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35237 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9fe0640122ec,34019,1733948023445
2024-12-11T20:13:43,683 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35237 {}] master.ServerManager(517): Registering regionserver=9fe0640122ec,34019,1733948023445
2024-12-11T20:13:43,685 DEBUG [RS:0;9fe0640122ec:34019 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b
2024-12-11T20:13:43,685 DEBUG [RS:0;9fe0640122ec:34019 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:46041
2024-12-11T20:13:43,685 DEBUG [RS:0;9fe0640122ec:34019 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-11T20:13:43,687 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35237-0x100caf934d80000, quorum=127.0.0.1:63467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-11T20:13:43,687 DEBUG [RS:0;9fe0640122ec:34019 {}] zookeeper.ZKUtil(111): regionserver:34019-0x100caf934d80001, quorum=127.0.0.1:63467, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9fe0640122ec,34019,1733948023445
2024-12-11T20:13:43,687 WARN  [RS:0;9fe0640122ec:34019 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-11T20:13:43,687 INFO  [RS:0;9fe0640122ec:34019 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-11T20:13:43,687 DEBUG [RS:0;9fe0640122ec:34019 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/WALs/9fe0640122ec,34019,1733948023445
2024-12-11T20:13:43,688 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9fe0640122ec,34019,1733948023445]
2024-12-11T20:13:43,691 INFO  [RS:0;9fe0640122ec:34019 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-11T20:13:43,692 INFO  [RS:0;9fe0640122ec:34019 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-11T20:13:43,693 INFO  [RS:0;9fe0640122ec:34019 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-11T20:13:43,693 INFO  [RS:0;9fe0640122ec:34019 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-11T20:13:43,693 INFO  [RS:0;9fe0640122ec:34019 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-11T20:13:43,694 INFO  [RS:0;9fe0640122ec:34019 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-11T20:13:43,694 INFO  [RS:0;9fe0640122ec:34019 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-11T20:13:43,694 DEBUG [RS:0;9fe0640122ec:34019 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:13:43,694 DEBUG [RS:0;9fe0640122ec:34019 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:13:43,694 DEBUG [RS:0;9fe0640122ec:34019 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:13:43,694 DEBUG [RS:0;9fe0640122ec:34019 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:13:43,694 DEBUG [RS:0;9fe0640122ec:34019 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:13:43,694 DEBUG [RS:0;9fe0640122ec:34019 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9fe0640122ec:0, corePoolSize=2, maxPoolSize=2
2024-12-11T20:13:43,694 DEBUG [RS:0;9fe0640122ec:34019 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:13:43,694 DEBUG [RS:0;9fe0640122ec:34019 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:13:43,694 DEBUG [RS:0;9fe0640122ec:34019 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:13:43,694 DEBUG [RS:0;9fe0640122ec:34019 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:13:43,694 DEBUG [RS:0;9fe0640122ec:34019 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:13:43,694 DEBUG [RS:0;9fe0640122ec:34019 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:13:43,694 DEBUG [RS:0;9fe0640122ec:34019 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9fe0640122ec:0, corePoolSize=3, maxPoolSize=3
2024-12-11T20:13:43,694 DEBUG [RS:0;9fe0640122ec:34019 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9fe0640122ec:0, corePoolSize=3, maxPoolSize=3
2024-12-11T20:13:43,696 INFO  [RS:0;9fe0640122ec:34019 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-11T20:13:43,696 INFO  [RS:0;9fe0640122ec:34019 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-11T20:13:43,696 INFO  [RS:0;9fe0640122ec:34019 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-11T20:13:43,696 INFO  [RS:0;9fe0640122ec:34019 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-11T20:13:43,696 INFO  [RS:0;9fe0640122ec:34019 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-11T20:13:43,696 INFO  [RS:0;9fe0640122ec:34019 {}] hbase.ChoreService(168): Chore ScheduledChore name=9fe0640122ec,34019,1733948023445-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-11T20:13:43,711 INFO  [RS:0;9fe0640122ec:34019 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-11T20:13:43,712 INFO  [RS:0;9fe0640122ec:34019 {}] hbase.ChoreService(168): Chore ScheduledChore name=9fe0640122ec,34019,1733948023445-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-11T20:13:43,712 INFO  [RS:0;9fe0640122ec:34019 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T20:13:43,712 INFO  [RS:0;9fe0640122ec:34019 {}] regionserver.Replication(171): 9fe0640122ec,34019,1733948023445 started
2024-12-11T20:13:43,726 INFO  [RS:0;9fe0640122ec:34019 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T20:13:43,726 INFO  [RS:0;9fe0640122ec:34019 {}] regionserver.HRegionServer(1482): Serving as 9fe0640122ec,34019,1733948023445, RpcServer on 9fe0640122ec/172.17.0.2:34019, sessionid=0x100caf934d80001
2024-12-11T20:13:43,726 DEBUG [RS:0;9fe0640122ec:34019 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-11T20:13:43,726 DEBUG [RS:0;9fe0640122ec:34019 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9fe0640122ec,34019,1733948023445
2024-12-11T20:13:43,727 DEBUG [RS:0;9fe0640122ec:34019 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9fe0640122ec,34019,1733948023445'
2024-12-11T20:13:43,727 DEBUG [RS:0;9fe0640122ec:34019 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-11T20:13:43,727 DEBUG [RS:0;9fe0640122ec:34019 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-11T20:13:43,728 DEBUG [RS:0;9fe0640122ec:34019 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-11T20:13:43,728 DEBUG [RS:0;9fe0640122ec:34019 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-11T20:13:43,728 DEBUG [RS:0;9fe0640122ec:34019 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9fe0640122ec,34019,1733948023445
2024-12-11T20:13:43,728 DEBUG [RS:0;9fe0640122ec:34019 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9fe0640122ec,34019,1733948023445'
2024-12-11T20:13:43,728 DEBUG [RS:0;9fe0640122ec:34019 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-11T20:13:43,728 DEBUG [RS:0;9fe0640122ec:34019 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-11T20:13:43,728 DEBUG [RS:0;9fe0640122ec:34019 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-11T20:13:43,728 INFO  [RS:0;9fe0640122ec:34019 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-11T20:13:43,728 INFO  [RS:0;9fe0640122ec:34019 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-11T20:13:43,780 WARN  [9fe0640122ec:35237 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-12-11T20:13:43,830 INFO  [RS:0;9fe0640122ec:34019 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9fe0640122ec%2C34019%2C1733948023445, suffix=, logDir=hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/WALs/9fe0640122ec,34019,1733948023445, archiveDir=hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/oldWALs, maxLogs=32
2024-12-11T20:13:43,831 INFO  [RS:0;9fe0640122ec:34019 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9fe0640122ec%2C34019%2C1733948023445.1733948023831
2024-12-11T20:13:43,836 INFO  [RS:0;9fe0640122ec:34019 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/WALs/9fe0640122ec,34019,1733948023445/9fe0640122ec%2C34019%2C1733948023445.1733948023831
2024-12-11T20:13:43,837 DEBUG [RS:0;9fe0640122ec:34019 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44059:44059),(127.0.0.1/127.0.0.1:37643:37643)]
2024-12-11T20:13:44,031 DEBUG [9fe0640122ec:35237 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-11T20:13:44,031 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=9fe0640122ec,34019,1733948023445
2024-12-11T20:13:44,033 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9fe0640122ec,34019,1733948023445, state=OPENING
2024-12-11T20:13:44,034 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-11T20:13:44,036 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35237-0x100caf934d80000, quorum=127.0.0.1:63467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:13:44,036 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34019-0x100caf934d80001, quorum=127.0.0.1:63467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:13:44,036 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-11T20:13:44,036 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-11T20:13:44,036 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-11T20:13:44,037 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=9fe0640122ec,34019,1733948023445}]
2024-12-11T20:13:44,190 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-11T20:13:44,192 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55313, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-11T20:13:44,196 INFO  [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-12-11T20:13:44,196 INFO  [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-11T20:13:44,198 INFO  [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9fe0640122ec%2C34019%2C1733948023445.meta, suffix=.meta, logDir=hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/WALs/9fe0640122ec,34019,1733948023445, archiveDir=hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/oldWALs, maxLogs=32
2024-12-11T20:13:44,198 INFO  [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 9fe0640122ec%2C34019%2C1733948023445.meta.1733948024198.meta
2024-12-11T20:13:44,204 INFO  [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/WALs/9fe0640122ec,34019,1733948023445/9fe0640122ec%2C34019%2C1733948023445.meta.1733948024198.meta
2024-12-11T20:13:44,205 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37643:37643),(127.0.0.1/127.0.0.1:44059:44059)]
2024-12-11T20:13:44,207 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-11T20:13:44,208 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-11T20:13:44,208 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-11T20:13:44,208 INFO  [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-11T20:13:44,208 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-11T20:13:44,208 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T20:13:44,209 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-12-11T20:13:44,209 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-12-11T20:13:44,210 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-11T20:13:44,211 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-11T20:13:44,211 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:13:44,212 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T20:13:44,212 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-11T20:13:44,212 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-11T20:13:44,212 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:13:44,213 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T20:13:44,213 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-11T20:13:44,214 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-11T20:13:44,214 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:13:44,214 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T20:13:44,214 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-11T20:13:44,215 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-11T20:13:44,215 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:13:44,215 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T20:13:44,215 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-11T20:13:44,216 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/data/hbase/meta/1588230740
2024-12-11T20:13:44,217 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/data/hbase/meta/1588230740
2024-12-11T20:13:44,218 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-11T20:13:44,218 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-11T20:13:44,218 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-11T20:13:44,220 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-11T20:13:44,221 INFO  [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=863265, jitterRate=0.09769895672798157}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-11T20:13:44,221 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-12-11T20:13:44,221 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1733948024209Writing region info on filesystem at 1733948024209Initializing all the Stores at 1733948024210 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733948024210Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733948024210Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733948024210Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733948024210Cleaning up temporary data from old regions at 1733948024218 (+8 ms)Running coprocessor post-open hooks at 1733948024221 (+3 ms)Region opened successfully at 1733948024221
2024-12-11T20:13:44,223 INFO  [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733948024189
2024-12-11T20:13:44,225 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-11T20:13:44,225 INFO  [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-12-11T20:13:44,226 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=9fe0640122ec,34019,1733948023445
2024-12-11T20:13:44,227 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9fe0640122ec,34019,1733948023445, state=OPEN
2024-12-11T20:13:44,231 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35237-0x100caf934d80000, quorum=127.0.0.1:63467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-11T20:13:44,231 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34019-0x100caf934d80001, quorum=127.0.0.1:63467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-11T20:13:44,231 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=9fe0640122ec,34019,1733948023445
2024-12-11T20:13:44,231 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-11T20:13:44,231 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-11T20:13:44,233 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-12-11T20:13:44,234 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9fe0640122ec,34019,1733948023445 in 195 msec
2024-12-11T20:13:44,237 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-12-11T20:13:44,237 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 606 msec
2024-12-11T20:13:44,238 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-11T20:13:44,238 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-12-11T20:13:44,239 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-11T20:13:44,239 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9fe0640122ec,34019,1733948023445, seqNum=-1]
2024-12-11T20:13:44,239 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-11T20:13:44,241 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-13-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39975, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-11T20:13:44,247 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 654 msec
2024-12-11T20:13:44,247 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733948024247, completionTime=-1
2024-12-11T20:13:44,247 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-11T20:13:44,247 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-12-11T20:13:44,249 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-12-11T20:13:44,249 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733948084249
2024-12-11T20:13:44,249 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733948144249
2024-12-11T20:13:44,249 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 1 msec
2024-12-11T20:13:44,249 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9fe0640122ec,35237,1733948023402-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-11T20:13:44,249 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9fe0640122ec,35237,1733948023402-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T20:13:44,249 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9fe0640122ec,35237,1733948023402-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T20:13:44,249 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-9fe0640122ec:35237, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T20:13:44,249 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-11T20:13:44,249 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-12-11T20:13:44,251 DEBUG [master/9fe0640122ec:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-12-11T20:13:44,253 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.773sec
2024-12-11T20:13:44,253 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-11T20:13:44,253 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-11T20:13:44,253 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-11T20:13:44,253 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-11T20:13:44,253 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-11T20:13:44,253 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9fe0640122ec,35237,1733948023402-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-11T20:13:44,253 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9fe0640122ec,35237,1733948023402-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-11T20:13:44,255 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-12-11T20:13:44,255 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-11T20:13:44,255 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9fe0640122ec,35237,1733948023402-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T20:13:44,264 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62aa92bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-11T20:13:44,264 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9fe0640122ec,35237,-1 for getting cluster id
2024-12-11T20:13:44,264 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-12-11T20:13:44,266 DEBUG [HMaster-EventLoopGroup-12-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dc791c5c-573b-4a4c-a995-329f7178dd77'
2024-12-11T20:13:44,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-12-11T20:13:44,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dc791c5c-573b-4a4c-a995-329f7178dd77"

2024-12-11T20:13:44,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4250eca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-11T20:13:44,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9fe0640122ec,35237,-1]
2024-12-11T20:13:44,267 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-12-11T20:13:44,267 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T20:13:44,268 INFO  [HMaster-EventLoopGroup-12-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43932, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-12-11T20:13:44,269 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5894e22d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-11T20:13:44,269 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-11T20:13:44,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9fe0640122ec,34019,1733948023445, seqNum=-1]
2024-12-11T20:13:44,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-11T20:13:44,271 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48436, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-11T20:13:44,273 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=9fe0640122ec,35237,1733948023402
2024-12-11T20:13:44,273 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T20:13:44,276 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-12-11T20:13:44,276 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-12-11T20:13:44,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncConnectionImpl(321): The fetched master address is 9fe0640122ec,35237,1733948023402
2024-12-11T20:13:44,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4a9663e3
2024-12-11T20:13:44,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-12-11T20:13:44,278 INFO  [HMaster-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43936, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-12-11T20:13:44,278 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35237 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-12-11T20:13:44,278 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35237 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-12-11T20:13:44,279 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35237 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testCompactionRecordDoesntBlockRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-11T20:13:44,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-11T20:13:44,281 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-12-11T20:13:44,281 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:13:44,281 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35237 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testCompactionRecordDoesntBlockRolling"
 procId is: 4
2024-12-11T20:13:44,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-11T20:13:44,282 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-11T20:13:44,288 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43661 is added to blk_1073741835_1011 (size=405)
2024-12-11T20:13:44,288 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:44,289 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38159 is added to blk_1073741835_1011 (size=405)
2024-12-11T20:13:44,289 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:44,291 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 73ad3832418d54490bc2efefb33a9d64, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733948024278.73ad3832418d54490bc2efefb33a9d64.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testCompactionRecordDoesntBlockRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b
2024-12-11T20:13:44,298 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43661 is added to blk_1073741836_1012 (size=88)
2024-12-11T20:13:44,299 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38159 is added to blk_1073741836_1012 (size=88)
2024-12-11T20:13:44,299 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733948024278.73ad3832418d54490bc2efefb33a9d64.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T20:13:44,299 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1722): Closing 73ad3832418d54490bc2efefb33a9d64, disabling compactions & flushes
2024-12-11T20:13:44,299 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733948024278.73ad3832418d54490bc2efefb33a9d64.
2024-12-11T20:13:44,299 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733948024278.73ad3832418d54490bc2efefb33a9d64.
2024-12-11T20:13:44,299 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733948024278.73ad3832418d54490bc2efefb33a9d64. after waiting 0 ms
2024-12-11T20:13:44,299 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733948024278.73ad3832418d54490bc2efefb33a9d64.
2024-12-11T20:13:44,299 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733948024278.73ad3832418d54490bc2efefb33a9d64.
2024-12-11T20:13:44,300 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for 73ad3832418d54490bc2efefb33a9d64:
Waiting for close lock at 1733948024299Disabling compacts and flushes for region at 1733948024299Disabling writes for close at 1733948024299Writing region close event to WAL at 1733948024299Closed at 1733948024299
2024-12-11T20:13:44,301 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ADD_TO_META
2024-12-11T20:13:44,301 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733948024278.73ad3832418d54490bc2efefb33a9d64.","families":{"info":[{"qualifier":"regioninfo","vlen":87,"tag":[],"timestamp":"1733948024301"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733948024301"}]},"ts":"1733948024301"}
2024-12-11T20:13:44,303 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-12-11T20:13:44,305 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-11T20:13:44,305 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733948024305"}]},"ts":"1733948024305"}
2024-12-11T20:13:44,307 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLING in hbase:meta
2024-12-11T20:13:44,307 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=73ad3832418d54490bc2efefb33a9d64, ASSIGN}]
2024-12-11T20:13:44,309 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=73ad3832418d54490bc2efefb33a9d64, ASSIGN
2024-12-11T20:13:44,310 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=73ad3832418d54490bc2efefb33a9d64, ASSIGN; state=OFFLINE, location=9fe0640122ec,34019,1733948023445; forceNewPlan=false, retain=false
2024-12-11T20:13:44,461 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=73ad3832418d54490bc2efefb33a9d64, regionState=OPENING, regionLocation=9fe0640122ec,34019,1733948023445
2024-12-11T20:13:44,463 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-13-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=73ad3832418d54490bc2efefb33a9d64, ASSIGN because future has completed
2024-12-11T20:13:44,464 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 73ad3832418d54490bc2efefb33a9d64, server=9fe0640122ec,34019,1733948023445}]
2024-12-11T20:13:44,620 INFO  [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733948024278.73ad3832418d54490bc2efefb33a9d64.
2024-12-11T20:13:44,620 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 73ad3832418d54490bc2efefb33a9d64, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733948024278.73ad3832418d54490bc2efefb33a9d64.', STARTKEY => '', ENDKEY => ''}
2024-12-11T20:13:44,621 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testCompactionRecordDoesntBlockRolling 73ad3832418d54490bc2efefb33a9d64
2024-12-11T20:13:44,621 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733948024278.73ad3832418d54490bc2efefb33a9d64.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T20:13:44,621 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 73ad3832418d54490bc2efefb33a9d64
2024-12-11T20:13:44,621 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 73ad3832418d54490bc2efefb33a9d64
2024-12-11T20:13:44,622 INFO  [StoreOpener-73ad3832418d54490bc2efefb33a9d64-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 73ad3832418d54490bc2efefb33a9d64 
2024-12-11T20:13:44,623 INFO  [StoreOpener-73ad3832418d54490bc2efefb33a9d64-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 73ad3832418d54490bc2efefb33a9d64 columnFamilyName info
2024-12-11T20:13:44,624 DEBUG [StoreOpener-73ad3832418d54490bc2efefb33a9d64-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:13:44,624 INFO  [StoreOpener-73ad3832418d54490bc2efefb33a9d64-1 {}] regionserver.HStore(327): Store=73ad3832418d54490bc2efefb33a9d64/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-11T20:13:44,624 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 73ad3832418d54490bc2efefb33a9d64
2024-12-11T20:13:44,625 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/73ad3832418d54490bc2efefb33a9d64
2024-12-11T20:13:44,625 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/73ad3832418d54490bc2efefb33a9d64
2024-12-11T20:13:44,625 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 73ad3832418d54490bc2efefb33a9d64
2024-12-11T20:13:44,626 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 73ad3832418d54490bc2efefb33a9d64
2024-12-11T20:13:44,627 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 73ad3832418d54490bc2efefb33a9d64
2024-12-11T20:13:44,629 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/73ad3832418d54490bc2efefb33a9d64/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-11T20:13:44,629 INFO  [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 73ad3832418d54490bc2efefb33a9d64; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=770221, jitterRate=-0.020613759756088257}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-11T20:13:44,629 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 73ad3832418d54490bc2efefb33a9d64
2024-12-11T20:13:44,630 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 73ad3832418d54490bc2efefb33a9d64:
Running coprocessor pre-open hook at 1733948024621Writing region info on filesystem at 1733948024621Initializing all the Stores at 1733948024622 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733948024622Cleaning up temporary data from old regions at 1733948024626 (+4 ms)Running coprocessor post-open hooks at 1733948024629 (+3 ms)Region opened successfully at 1733948024630 (+1 ms)
2024-12-11T20:13:44,631 INFO  [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733948024278.73ad3832418d54490bc2efefb33a9d64., pid=6, masterSystemTime=1733948024616
2024-12-11T20:13:44,634 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733948024278.73ad3832418d54490bc2efefb33a9d64.
2024-12-11T20:13:44,634 INFO  [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733948024278.73ad3832418d54490bc2efefb33a9d64.
2024-12-11T20:13:44,635 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=73ad3832418d54490bc2efefb33a9d64, regionState=OPEN, openSeqNum=2, regionLocation=9fe0640122ec,34019,1733948023445
2024-12-11T20:13:44,637 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-13-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 73ad3832418d54490bc2efefb33a9d64, server=9fe0640122ec,34019,1733948023445 because future has completed
2024-12-11T20:13:44,641 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-12-11T20:13:44,641 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 73ad3832418d54490bc2efefb33a9d64, server=9fe0640122ec,34019,1733948023445 in 175 msec
2024-12-11T20:13:44,644 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-12-11T20:13:44,644 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=73ad3832418d54490bc2efefb33a9d64, ASSIGN in 334 msec
2024-12-11T20:13:44,645 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-11T20:13:44,645 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733948024645"}]},"ts":"1733948024645"}
2024-12-11T20:13:44,647 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLED in hbase:meta
2024-12-11T20:13:44,648 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_POST_OPERATION
2024-12-11T20:13:44,650 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 369 msec
2024-12-11T20:13:45,289 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:45,289 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:46,290 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:46,290 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:47,290 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:47,290 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:47,658 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-11T20:13:47,659 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:47,659 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:47,659 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:47,659 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:47,660 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:47,660 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:47,673 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:47,673 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:47,674 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:47,674 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:47,674 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:47,674 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:47,677 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:47,677 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:47,678 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:47,680 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:13:48,291 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:48,291 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:49,292 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:49,292 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:49,691 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-12-11T20:13:49,692 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testCompactionRecordDoesntBlockRolling'
2024-12-11T20:13:50,293 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:50,293 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:51,293 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:51,293 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:52,156 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-12-11T20:13:52,156 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-12-11T20:13:52,156 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-11T20:13:52,156 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-12-11T20:13:52,157 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-12-11T20:13:52,157 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-12-11T20:13:52,157 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-11T20:13:52,157 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling Metrics about Tables on a single HBase RegionServer
2024-12-11T20:13:52,294 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:52,294 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:53,295 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:53,295 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:54,295 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:54,295 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:54,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-11T20:13:54,367 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-12-11T20:13:54,367 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testCompactionRecordDoesntBlockRolling,, stopping at row=TestLogRolling-testCompactionRecordDoesntBlockRolling ,, for max=2147483647 with caching=100
2024-12-11T20:13:54,370 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-11T20:13:54,370 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733948024278.73ad3832418d54490bc2efefb33a9d64.
2024-12-11T20:13:54,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testCompactionRecordDoesntBlockRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733948024278.73ad3832418d54490bc2efefb33a9d64., hostname=9fe0640122ec,34019,1733948023445, seqNum=2]
2024-12-11T20:13:54,380 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35237 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-11T20:13:54,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-11T20:13:54,386 INFO  [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-12-11T20:13:54,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7
2024-12-11T20:13:54,387 INFO  [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-12-11T20:13:54,389 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-12-11T20:13:54,549 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34019 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8
2024-12-11T20:13:54,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9fe0640122ec:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733948024278.73ad3832418d54490bc2efefb33a9d64.
2024-12-11T20:13:54,550 INFO  [RS_FLUSH_OPERATIONS-regionserver/9fe0640122ec:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 73ad3832418d54490bc2efefb33a9d64 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-12-11T20:13:54,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9fe0640122ec:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/73ad3832418d54490bc2efefb33a9d64/.tmp/info/13d0392f08f546aba72aaffbb36c59dd is 1080, key is row0001/info:/1733948034374/Put/seqid=0
2024-12-11T20:13:54,572 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38159 is added to blk_1073741837_1013 (size=6033)
2024-12-11T20:13:54,573 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43661 is added to blk_1073741837_1013 (size=6033)
2024-12-11T20:13:54,573 INFO  [RS_FLUSH_OPERATIONS-regionserver/9fe0640122ec:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/73ad3832418d54490bc2efefb33a9d64/.tmp/info/13d0392f08f546aba72aaffbb36c59dd
2024-12-11T20:13:54,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9fe0640122ec:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/73ad3832418d54490bc2efefb33a9d64/.tmp/info/13d0392f08f546aba72aaffbb36c59dd as hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/73ad3832418d54490bc2efefb33a9d64/info/13d0392f08f546aba72aaffbb36c59dd
2024-12-11T20:13:54,584 INFO  [RS_FLUSH_OPERATIONS-regionserver/9fe0640122ec:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/73ad3832418d54490bc2efefb33a9d64/info/13d0392f08f546aba72aaffbb36c59dd, entries=1, sequenceid=5, filesize=5.9 K
2024-12-11T20:13:54,585 INFO  [RS_FLUSH_OPERATIONS-regionserver/9fe0640122ec:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 73ad3832418d54490bc2efefb33a9d64 in 35ms, sequenceid=5, compaction requested=false
2024-12-11T20:13:54,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9fe0640122ec:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 73ad3832418d54490bc2efefb33a9d64:

2024-12-11T20:13:54,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9fe0640122ec:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733948024278.73ad3832418d54490bc2efefb33a9d64.
2024-12-11T20:13:54,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9fe0640122ec:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8
2024-12-11T20:13:54,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35237 {}] master.HMaster(4169): Remote procedure done, pid=8
2024-12-11T20:13:54,593 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7
2024-12-11T20:13:54,593 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 201 msec
2024-12-11T20:13:54,595 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 212 msec
2024-12-11T20:13:55,296 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:55,296 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:56,297 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:56,297 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:57,297 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:57,297 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:58,298 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:58,298 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:59,298 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:13:59,298 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:00,299 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:00,299 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:01,300 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:01,300 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:02,300 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:02,300 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:03,301 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:03,301 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:04,302 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:04,302 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:04,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7
2024-12-11T20:14:04,418 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-12-11T20:14:04,421 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35237 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-11T20:14:04,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-11T20:14:04,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9
2024-12-11T20:14:04,423 INFO  [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-12-11T20:14:04,424 INFO  [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-12-11T20:14:04,424 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-12-11T20:14:04,577 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34019 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10
2024-12-11T20:14:04,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9fe0640122ec:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733948024278.73ad3832418d54490bc2efefb33a9d64.
2024-12-11T20:14:04,578 INFO  [RS_FLUSH_OPERATIONS-regionserver/9fe0640122ec:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 73ad3832418d54490bc2efefb33a9d64 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-12-11T20:14:04,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9fe0640122ec:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/73ad3832418d54490bc2efefb33a9d64/.tmp/info/5791cf1e19ba41d9ae0d8bec141520c6 is 1080, key is row0002/info:/1733948044419/Put/seqid=0
2024-12-11T20:14:04,587 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43661 is added to blk_1073741838_1014 (size=6033)
2024-12-11T20:14:04,588 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38159 is added to blk_1073741838_1014 (size=6033)
2024-12-11T20:14:04,588 INFO  [RS_FLUSH_OPERATIONS-regionserver/9fe0640122ec:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/73ad3832418d54490bc2efefb33a9d64/.tmp/info/5791cf1e19ba41d9ae0d8bec141520c6
2024-12-11T20:14:04,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9fe0640122ec:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/73ad3832418d54490bc2efefb33a9d64/.tmp/info/5791cf1e19ba41d9ae0d8bec141520c6 as hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/73ad3832418d54490bc2efefb33a9d64/info/5791cf1e19ba41d9ae0d8bec141520c6
2024-12-11T20:14:04,601 INFO  [RS_FLUSH_OPERATIONS-regionserver/9fe0640122ec:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/73ad3832418d54490bc2efefb33a9d64/info/5791cf1e19ba41d9ae0d8bec141520c6, entries=1, sequenceid=9, filesize=5.9 K
2024-12-11T20:14:04,602 INFO  [RS_FLUSH_OPERATIONS-regionserver/9fe0640122ec:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 73ad3832418d54490bc2efefb33a9d64 in 24ms, sequenceid=9, compaction requested=false
2024-12-11T20:14:04,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9fe0640122ec:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 73ad3832418d54490bc2efefb33a9d64:

2024-12-11T20:14:04,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9fe0640122ec:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733948024278.73ad3832418d54490bc2efefb33a9d64.
2024-12-11T20:14:04,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9fe0640122ec:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10
2024-12-11T20:14:04,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35237 {}] master.HMaster(4169): Remote procedure done, pid=10
2024-12-11T20:14:04,607 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9
2024-12-11T20:14:04,607 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 180 msec
2024-12-11T20:14:04,609 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 186 msec
2024-12-11T20:14:05,302 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:05,302 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:06,303 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:06,303 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:07,303 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:07,303 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:08,304 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:08,304 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:08,305 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta after 68038ms
java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor193.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:14:08,305 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403 after 68050ms
java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor193.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-11T20:14:09,305 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:09,305 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:10,306 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:10,306 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:11,306 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:11,306 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:12,307 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:12,307 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:13,308 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:13,308 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:13,383 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-11T20:14:14,308 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:14,308 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:14,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9
2024-12-11T20:14:14,458 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-12-11T20:14:14,461 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9fe0640122ec%2C34019%2C1733948023445.1733948054461
2024-12-11T20:14:14,466 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:14:14,466 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:14:14,466 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:14:14,466 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:14:14,466 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:14:14,467 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/WALs/9fe0640122ec,34019,1733948023445/9fe0640122ec%2C34019%2C1733948023445.1733948023831 with entries=8, filesize=5.41 KB; new WAL /user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/WALs/9fe0640122ec,34019,1733948023445/9fe0640122ec%2C34019%2C1733948023445.1733948054461
2024-12-11T20:14:14,469 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38159 is added to blk_1073741833_1009 (size=5546)
2024-12-11T20:14:14,469 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43661 is added to blk_1073741833_1009 (size=5546)
2024-12-11T20:14:14,474 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44059:44059),(127.0.0.1/127.0.0.1:37643:37643)]
2024-12-11T20:14:14,475 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35237 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-11T20:14:14,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-11T20:14:14,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11
2024-12-11T20:14:14,478 INFO  [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-12-11T20:14:14,479 INFO  [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-12-11T20:14:14,479 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-12-11T20:14:14,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34019 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12
2024-12-11T20:14:14,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9fe0640122ec:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733948024278.73ad3832418d54490bc2efefb33a9d64.
2024-12-11T20:14:14,633 INFO  [RS_FLUSH_OPERATIONS-regionserver/9fe0640122ec:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 73ad3832418d54490bc2efefb33a9d64 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-12-11T20:14:14,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9fe0640122ec:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/73ad3832418d54490bc2efefb33a9d64/.tmp/info/d276cdeb7f154613bb6cf8ea7bccc280 is 1080, key is row0003/info:/1733948054459/Put/seqid=0
2024-12-11T20:14:14,642 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43661 is added to blk_1073741840_1016 (size=6033)
2024-12-11T20:14:14,643 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38159 is added to blk_1073741840_1016 (size=6033)
2024-12-11T20:14:14,643 INFO  [RS_FLUSH_OPERATIONS-regionserver/9fe0640122ec:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/73ad3832418d54490bc2efefb33a9d64/.tmp/info/d276cdeb7f154613bb6cf8ea7bccc280
2024-12-11T20:14:14,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9fe0640122ec:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/73ad3832418d54490bc2efefb33a9d64/.tmp/info/d276cdeb7f154613bb6cf8ea7bccc280 as hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/73ad3832418d54490bc2efefb33a9d64/info/d276cdeb7f154613bb6cf8ea7bccc280
2024-12-11T20:14:14,655 INFO  [RS_FLUSH_OPERATIONS-regionserver/9fe0640122ec:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/73ad3832418d54490bc2efefb33a9d64/info/d276cdeb7f154613bb6cf8ea7bccc280, entries=1, sequenceid=13, filesize=5.9 K
2024-12-11T20:14:14,657 INFO  [RS_FLUSH_OPERATIONS-regionserver/9fe0640122ec:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 73ad3832418d54490bc2efefb33a9d64 in 23ms, sequenceid=13, compaction requested=true
2024-12-11T20:14:14,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9fe0640122ec:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 73ad3832418d54490bc2efefb33a9d64:

2024-12-11T20:14:14,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9fe0640122ec:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733948024278.73ad3832418d54490bc2efefb33a9d64.
2024-12-11T20:14:14,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9fe0640122ec:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12
2024-12-11T20:14:14,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35237 {}] master.HMaster(4169): Remote procedure done, pid=12
2024-12-11T20:14:14,661 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11
2024-12-11T20:14:14,661 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 180 msec
2024-12-11T20:14:14,664 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 187 msec
2024-12-11T20:14:15,309 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:15,309 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:16,310 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:16,310 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:17,310 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:17,310 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:18,311 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:18,311 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:19,311 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:19,311 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:20,312 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:20,312 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:21,313 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:21,313 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:22,313 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:22,313 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:23,314 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:23,314 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:24,256 INFO  [master/9fe0640122ec:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-12-11T20:14:24,256 INFO  [master/9fe0640122ec:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-12-11T20:14:24,315 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:24,315 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:24,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11
2024-12-11T20:14:24,548 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-12-11T20:14:24,548 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-11T20:14:24,549 DEBUG [Time-limited test {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 18099 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-11T20:14:24,549 DEBUG [Time-limited test {}] regionserver.HStore(1541): 73ad3832418d54490bc2efefb33a9d64/info is initiating minor compaction (all files)
2024-12-11T20:14:24,549 INFO  [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-11T20:14:24,550 INFO  [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-11T20:14:24,550 INFO  [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 73ad3832418d54490bc2efefb33a9d64/info in TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733948024278.73ad3832418d54490bc2efefb33a9d64.
2024-12-11T20:14:24,550 INFO  [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/73ad3832418d54490bc2efefb33a9d64/info/13d0392f08f546aba72aaffbb36c59dd, hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/73ad3832418d54490bc2efefb33a9d64/info/5791cf1e19ba41d9ae0d8bec141520c6, hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/73ad3832418d54490bc2efefb33a9d64/info/d276cdeb7f154613bb6cf8ea7bccc280] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/73ad3832418d54490bc2efefb33a9d64/.tmp, totalSize=17.7 K
2024-12-11T20:14:24,550 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 13d0392f08f546aba72aaffbb36c59dd, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1733948034374
2024-12-11T20:14:24,551 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 5791cf1e19ba41d9ae0d8bec141520c6, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=9, earliestPutTs=1733948044419
2024-12-11T20:14:24,551 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting d276cdeb7f154613bb6cf8ea7bccc280, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733948054459
2024-12-11T20:14:24,563 INFO  [Time-limited test {}] throttle.PressureAwareThroughputController(145): 73ad3832418d54490bc2efefb33a9d64#info#compaction#46 average throughput is 3.08 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-11T20:14:24,563 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/73ad3832418d54490bc2efefb33a9d64/.tmp/info/5894b91c88844905997022cb282e0756 is 1080, key is row0001/info:/1733948034374/Put/seqid=0
2024-12-11T20:14:24,568 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43661 is added to blk_1073741841_1017 (size=8296)
2024-12-11T20:14:24,568 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38159 is added to blk_1073741841_1017 (size=8296)
2024-12-11T20:14:24,574 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/73ad3832418d54490bc2efefb33a9d64/.tmp/info/5894b91c88844905997022cb282e0756 as hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/73ad3832418d54490bc2efefb33a9d64/info/5894b91c88844905997022cb282e0756
2024-12-11T20:14:24,581 INFO  [Time-limited test {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 73ad3832418d54490bc2efefb33a9d64/info of 73ad3832418d54490bc2efefb33a9d64 into 5894b91c88844905997022cb282e0756(size=8.1 K), total size for store is 8.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-11T20:14:24,581 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 73ad3832418d54490bc2efefb33a9d64:

2024-12-11T20:14:24,583 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9fe0640122ec%2C34019%2C1733948023445.1733948064583
2024-12-11T20:14:24,589 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:14:24,589 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:14:24,589 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:14:24,589 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:14:24,589 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:14:24,589 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/WALs/9fe0640122ec,34019,1733948023445/9fe0640122ec%2C34019%2C1733948023445.1733948054461 with entries=4, filesize=2.45 KB; new WAL /user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/WALs/9fe0640122ec,34019,1733948023445/9fe0640122ec%2C34019%2C1733948023445.1733948064583
2024-12-11T20:14:24,590 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44059:44059),(127.0.0.1/127.0.0.1:37643:37643)]
2024-12-11T20:14:24,590 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/WALs/9fe0640122ec,34019,1733948023445/9fe0640122ec%2C34019%2C1733948023445.1733948054461 is not closed yet, will try archiving it next time
2024-12-11T20:14:24,591 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/WALs/9fe0640122ec,34019,1733948023445/9fe0640122ec%2C34019%2C1733948023445.1733948023831 to hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/oldWALs/9fe0640122ec%2C34019%2C1733948023445.1733948023831
2024-12-11T20:14:24,591 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38159 is added to blk_1073741839_1015 (size=2520)
2024-12-11T20:14:24,591 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43661 is added to blk_1073741839_1015 (size=2520)
2024-12-11T20:14:24,591 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35237 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-11T20:14:24,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35237 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-11T20:14:24,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13
2024-12-11T20:14:24,594 INFO  [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-12-11T20:14:24,594 INFO  [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-12-11T20:14:24,595 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-12-11T20:14:24,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34019 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14
2024-12-11T20:14:24,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9fe0640122ec:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733948024278.73ad3832418d54490bc2efefb33a9d64.
2024-12-11T20:14:24,748 INFO  [RS_FLUSH_OPERATIONS-regionserver/9fe0640122ec:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 73ad3832418d54490bc2efefb33a9d64 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-12-11T20:14:24,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9fe0640122ec:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/73ad3832418d54490bc2efefb33a9d64/.tmp/info/b9a1b71a8b344beaacdf4bc3848b9311 is 1080, key is row0000/info:/1733948064582/Put/seqid=0
2024-12-11T20:14:24,757 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43661 is added to blk_1073741843_1019 (size=6033)
2024-12-11T20:14:24,757 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38159 is added to blk_1073741843_1019 (size=6033)
2024-12-11T20:14:24,757 INFO  [RS_FLUSH_OPERATIONS-regionserver/9fe0640122ec:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/73ad3832418d54490bc2efefb33a9d64/.tmp/info/b9a1b71a8b344beaacdf4bc3848b9311
2024-12-11T20:14:24,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9fe0640122ec:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/73ad3832418d54490bc2efefb33a9d64/.tmp/info/b9a1b71a8b344beaacdf4bc3848b9311 as hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/73ad3832418d54490bc2efefb33a9d64/info/b9a1b71a8b344beaacdf4bc3848b9311
2024-12-11T20:14:24,768 INFO  [RS_FLUSH_OPERATIONS-regionserver/9fe0640122ec:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/73ad3832418d54490bc2efefb33a9d64/info/b9a1b71a8b344beaacdf4bc3848b9311, entries=1, sequenceid=18, filesize=5.9 K
2024-12-11T20:14:24,769 INFO  [RS_FLUSH_OPERATIONS-regionserver/9fe0640122ec:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 73ad3832418d54490bc2efefb33a9d64 in 21ms, sequenceid=18, compaction requested=false
2024-12-11T20:14:24,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9fe0640122ec:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 73ad3832418d54490bc2efefb33a9d64:

2024-12-11T20:14:24,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9fe0640122ec:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733948024278.73ad3832418d54490bc2efefb33a9d64.
2024-12-11T20:14:24,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9fe0640122ec:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14
2024-12-11T20:14:24,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35237 {}] master.HMaster(4169): Remote procedure done, pid=14
2024-12-11T20:14:24,774 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13
2024-12-11T20:14:24,774 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 177 msec
2024-12-11T20:14:24,776 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 184 msec
2024-12-11T20:14:25,315 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:25,315 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:26,316 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:26,316 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:27,316 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:27,316 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:28,317 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:28,317 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:29,318 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:29,318 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:29,621 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 73ad3832418d54490bc2efefb33a9d64, had cached 0 bytes from a total of 14329
2024-12-11T20:14:30,318 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:30,318 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:31,319 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:31,319 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:32,320 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:32,320 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:33,320 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:33,320 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:34,321 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:34,321 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:34,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13
2024-12-11T20:14:34,627 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-12-11T20:14:34,630 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9fe0640122ec%2C34019%2C1733948023445.1733948074630
2024-12-11T20:14:34,636 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:14:34,636 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:14:34,636 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:14:34,636 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:14:34,636 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:14:34,636 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/WALs/9fe0640122ec,34019,1733948023445/9fe0640122ec%2C34019%2C1733948023445.1733948064583 with entries=3, filesize=1.97 KB; new WAL /user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/WALs/9fe0640122ec,34019,1733948023445/9fe0640122ec%2C34019%2C1733948023445.1733948074630
2024-12-11T20:14:34,637 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44059:44059),(127.0.0.1/127.0.0.1:37643:37643)]
2024-12-11T20:14:34,637 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/WALs/9fe0640122ec,34019,1733948023445/9fe0640122ec%2C34019%2C1733948023445.1733948064583 is not closed yet, will try archiving it next time
2024-12-11T20:14:34,637 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-12-11T20:14:34,637 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/WALs/9fe0640122ec,34019,1733948023445/9fe0640122ec%2C34019%2C1733948023445.1733948054461 to hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/oldWALs/9fe0640122ec%2C34019%2C1733948023445.1733948054461
2024-12-11T20:14:34,637 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-11T20:14:34,638 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-11T20:14:34,638 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T20:14:34,638 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T20:14:34,638 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38159 is added to blk_1073741842_1018 (size=2026)
2024-12-11T20:14:34,638 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-11T20:14:34,638 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1474391260, stopped=false
2024-12-11T20:14:34,638 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43661 is added to blk_1073741842_1018 (size=2026)
2024-12-11T20:14:34,638 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=9fe0640122ec,35237,1733948023402
2024-12-11T20:14:34,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35237-0x100caf934d80000, quorum=127.0.0.1:63467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-11T20:14:34,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34019-0x100caf934d80001, quorum=127.0.0.1:63467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-11T20:14:34,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35237-0x100caf934d80000, quorum=127.0.0.1:63467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:14:34,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34019-0x100caf934d80001, quorum=127.0.0.1:63467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:14:34,640 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-11T20:14:34,640 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-11T20:14:34,640 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-11T20:14:34,641 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T20:14:34,641 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9fe0640122ec,34019,1733948023445' *****
2024-12-11T20:14:34,641 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-12-11T20:14:34,641 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35237-0x100caf934d80000, quorum=127.0.0.1:63467, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-11T20:14:34,641 INFO  [RS:0;9fe0640122ec:34019 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-11T20:14:34,641 INFO  [RS:0;9fe0640122ec:34019 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-11T20:14:34,641 INFO  [RS:0;9fe0640122ec:34019 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-11T20:14:34,641 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-12-11T20:14:34,641 INFO  [RS:0;9fe0640122ec:34019 {}] regionserver.HRegionServer(3091): Received CLOSE for 73ad3832418d54490bc2efefb33a9d64
2024-12-11T20:14:34,641 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34019-0x100caf934d80001, quorum=127.0.0.1:63467, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-11T20:14:34,642 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-12-11T20:14:34,642 INFO  [RS:0;9fe0640122ec:34019 {}] regionserver.HRegionServer(959): stopping server 9fe0640122ec,34019,1733948023445
2024-12-11T20:14:34,642 INFO  [RS:0;9fe0640122ec:34019 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-11T20:14:34,642 INFO  [RS:0;9fe0640122ec:34019 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;9fe0640122ec:34019.
2024-12-11T20:14:34,642 DEBUG [RS:0;9fe0640122ec:34019 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-11T20:14:34,642 DEBUG [RS:0;9fe0640122ec:34019 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T20:14:34,642 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 73ad3832418d54490bc2efefb33a9d64, disabling compactions & flushes
2024-12-11T20:14:34,642 INFO  [RS:0;9fe0640122ec:34019 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-11T20:14:34,642 INFO  [RS:0;9fe0640122ec:34019 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-11T20:14:34,642 INFO  [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733948024278.73ad3832418d54490bc2efefb33a9d64.
2024-12-11T20:14:34,642 INFO  [RS:0;9fe0640122ec:34019 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-11T20:14:34,642 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733948024278.73ad3832418d54490bc2efefb33a9d64.
2024-12-11T20:14:34,642 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733948024278.73ad3832418d54490bc2efefb33a9d64. after waiting 0 ms
2024-12-11T20:14:34,642 INFO  [RS:0;9fe0640122ec:34019 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-12-11T20:14:34,642 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733948024278.73ad3832418d54490bc2efefb33a9d64.
2024-12-11T20:14:34,643 INFO  [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 73ad3832418d54490bc2efefb33a9d64 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-12-11T20:14:34,643 INFO  [RS:0;9fe0640122ec:34019 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-12-11T20:14:34,643 DEBUG [RS:0;9fe0640122ec:34019 {}] regionserver.HRegionServer(1325): Online Regions={73ad3832418d54490bc2efefb33a9d64=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733948024278.73ad3832418d54490bc2efefb33a9d64., 1588230740=hbase:meta,,1.1588230740}
2024-12-11T20:14:34,643 DEBUG [RS:0;9fe0640122ec:34019 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 73ad3832418d54490bc2efefb33a9d64
2024-12-11T20:14:34,643 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-11T20:14:34,643 INFO  [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-11T20:14:34,643 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-11T20:14:34,643 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-11T20:14:34,643 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-11T20:14:34,643 INFO  [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.89 KB heapSize=3.91 KB
2024-12-11T20:14:34,649 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/73ad3832418d54490bc2efefb33a9d64/.tmp/info/3da5b952cf364d09ae9372d05607aa15 is 1080, key is row0001/info:/1733948074628/Put/seqid=0
2024-12-11T20:14:34,654 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38159 is added to blk_1073741845_1021 (size=6033)
2024-12-11T20:14:34,654 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43661 is added to blk_1073741845_1021 (size=6033)
2024-12-11T20:14:34,663 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/data/hbase/meta/1588230740/.tmp/info/68e31689ea144c6194e49b5dbd57851b is 227, key is TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733948024278.73ad3832418d54490bc2efefb33a9d64./info:regioninfo/1733948024635/Put/seqid=0
2024-12-11T20:14:34,668 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38159 is added to blk_1073741846_1022 (size=7308)
2024-12-11T20:14:34,668 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43661 is added to blk_1073741846_1022 (size=7308)
2024-12-11T20:14:34,669 INFO  [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/data/hbase/meta/1588230740/.tmp/info/68e31689ea144c6194e49b5dbd57851b
2024-12-11T20:14:34,688 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/data/hbase/meta/1588230740/.tmp/ns/9a7fb997e23d433e889b13652804dffc is 43, key is default/ns:d/1733948024241/Put/seqid=0
2024-12-11T20:14:34,693 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43661 is added to blk_1073741847_1023 (size=5153)
2024-12-11T20:14:34,693 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38159 is added to blk_1073741847_1023 (size=5153)
2024-12-11T20:14:34,694 INFO  [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/data/hbase/meta/1588230740/.tmp/ns/9a7fb997e23d433e889b13652804dffc
2024-12-11T20:14:34,696 INFO  [regionserver/9fe0640122ec:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-12-11T20:14:34,696 INFO  [regionserver/9fe0640122ec:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-12-11T20:14:34,714 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/data/hbase/meta/1588230740/.tmp/table/640760d4c38f4f2fb8e8db39c19cb871 is 89, key is TestLogRolling-testCompactionRecordDoesntBlockRolling/table:state/1733948024645/Put/seqid=0
2024-12-11T20:14:34,718 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38159 is added to blk_1073741848_1024 (size=5508)
2024-12-11T20:14:34,718 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43661 is added to blk_1073741848_1024 (size=5508)
2024-12-11T20:14:34,719 INFO  [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=170 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/data/hbase/meta/1588230740/.tmp/table/640760d4c38f4f2fb8e8db39c19cb871
2024-12-11T20:14:34,724 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/data/hbase/meta/1588230740/.tmp/info/68e31689ea144c6194e49b5dbd57851b as hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/data/hbase/meta/1588230740/info/68e31689ea144c6194e49b5dbd57851b
2024-12-11T20:14:34,729 INFO  [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/data/hbase/meta/1588230740/info/68e31689ea144c6194e49b5dbd57851b, entries=10, sequenceid=11, filesize=7.1 K
2024-12-11T20:14:34,730 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/data/hbase/meta/1588230740/.tmp/ns/9a7fb997e23d433e889b13652804dffc as hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/data/hbase/meta/1588230740/ns/9a7fb997e23d433e889b13652804dffc
2024-12-11T20:14:34,735 INFO  [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/data/hbase/meta/1588230740/ns/9a7fb997e23d433e889b13652804dffc, entries=2, sequenceid=11, filesize=5.0 K
2024-12-11T20:14:34,735 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/data/hbase/meta/1588230740/.tmp/table/640760d4c38f4f2fb8e8db39c19cb871 as hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/data/hbase/meta/1588230740/table/640760d4c38f4f2fb8e8db39c19cb871
2024-12-11T20:14:34,740 INFO  [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/data/hbase/meta/1588230740/table/640760d4c38f4f2fb8e8db39c19cb871, entries=2, sequenceid=11, filesize=5.4 K
2024-12-11T20:14:34,741 INFO  [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.89 KB/1932, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 98ms, sequenceid=11, compaction requested=false
2024-12-11T20:14:34,745 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-12-11T20:14:34,746 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-11T20:14:34,746 INFO  [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-11T20:14:34,746 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733948074643Running coprocessor pre-close hooks at 1733948074643Disabling compacts and flushes for region at 1733948074643Disabling writes for close at 1733948074643Obtaining lock to block concurrent updates at 1733948074643Preparing flush snapshotting stores in 1588230740 at 1733948074643Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1932, getHeapSize=3936, getOffHeapSize=0, getCellsCount=14 at 1733948074643Flushing stores of hbase:meta,,1.1588230740 at 1733948074644 (+1 ms)Flushing 1588230740/info: creating writer at 1733948074644Flushing 1588230740/info: appending metadata at 1733948074663 (+19 ms)Flushing 1588230740/info: closing flushed file at 1733948074663Flushing 1588230740/ns: creating writer at 1733948074674 (+11 ms)Flushing 1588230740/ns: appending metadata at 1733948074688 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1733948074688Flushing 1588230740/table: creating writer at 1733948074699 (+11 ms)Flushing 1588230740/table: appending metadata at 1733948074713 (+14 ms)Flushing 1588230740/table: closing flushed file at 1733948074713Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5afb0cd2: reopening flushed file at 1733948074723 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7191eb8c: reopening flushed file at 1733948074729 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@950462: reopening flushed file at 1733948074735 (+6 ms)Finished flush of dataSize ~1.89 KB/1932, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 98ms, sequenceid=11, compaction requested=false at 1733948074741 (+6 ms)Writing region close event to WAL at 1733948074742 (+1 ms)Running coprocessor post-close hooks at 1733948074746 (+4 ms)Closed at 1733948074746
2024-12-11T20:14:34,746 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-11T20:14:34,843 DEBUG [RS:0;9fe0640122ec:34019 {}] regionserver.HRegionServer(1351): Waiting on 73ad3832418d54490bc2efefb33a9d64
2024-12-11T20:14:35,043 DEBUG [RS:0;9fe0640122ec:34019 {}] regionserver.HRegionServer(1351): Waiting on 73ad3832418d54490bc2efefb33a9d64
2024-12-11T20:14:35,055 INFO  [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=22 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/73ad3832418d54490bc2efefb33a9d64/.tmp/info/3da5b952cf364d09ae9372d05607aa15
2024-12-11T20:14:35,061 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/73ad3832418d54490bc2efefb33a9d64/.tmp/info/3da5b952cf364d09ae9372d05607aa15 as hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/73ad3832418d54490bc2efefb33a9d64/info/3da5b952cf364d09ae9372d05607aa15
2024-12-11T20:14:35,066 INFO  [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/73ad3832418d54490bc2efefb33a9d64/info/3da5b952cf364d09ae9372d05607aa15, entries=1, sequenceid=22, filesize=5.9 K
2024-12-11T20:14:35,067 INFO  [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 73ad3832418d54490bc2efefb33a9d64 in 425ms, sequenceid=22, compaction requested=true
2024-12-11T20:14:35,067 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733948024278.73ad3832418d54490bc2efefb33a9d64.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/73ad3832418d54490bc2efefb33a9d64/info/13d0392f08f546aba72aaffbb36c59dd, hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/73ad3832418d54490bc2efefb33a9d64/info/5791cf1e19ba41d9ae0d8bec141520c6, hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/73ad3832418d54490bc2efefb33a9d64/info/d276cdeb7f154613bb6cf8ea7bccc280] to archive
2024-12-11T20:14:35,068 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733948024278.73ad3832418d54490bc2efefb33a9d64.-1 {}] backup.HFileArchiver(363): Archiving compacted files.
2024-12-11T20:14:35,070 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/73ad3832418d54490bc2efefb33a9d64/info/13d0392f08f546aba72aaffbb36c59dd to hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/73ad3832418d54490bc2efefb33a9d64/info/13d0392f08f546aba72aaffbb36c59dd
2024-12-11T20:14:35,070 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/73ad3832418d54490bc2efefb33a9d64/info/5791cf1e19ba41d9ae0d8bec141520c6 to hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/73ad3832418d54490bc2efefb33a9d64/info/5791cf1e19ba41d9ae0d8bec141520c6
2024-12-11T20:14:35,070 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/73ad3832418d54490bc2efefb33a9d64/info/d276cdeb7f154613bb6cf8ea7bccc280 to hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/73ad3832418d54490bc2efefb33a9d64/info/d276cdeb7f154613bb6cf8ea7bccc280
2024-12-11T20:14:35,071 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733948024278.73ad3832418d54490bc2efefb33a9d64.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=9fe0640122ec:35237 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 16 more
2024-12-11T20:14:35,071 WARN  [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733948024278.73ad3832418d54490bc2efefb33a9d64.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [13d0392f08f546aba72aaffbb36c59dd=6033, 5791cf1e19ba41d9ae0d8bec141520c6=6033, d276cdeb7f154613bb6cf8ea7bccc280=6033]
2024-12-11T20:14:35,075 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/73ad3832418d54490bc2efefb33a9d64/recovered.edits/25.seqid, newMaxSeqId=25, maxSeqId=1
2024-12-11T20:14:35,075 INFO  [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733948024278.73ad3832418d54490bc2efefb33a9d64.
2024-12-11T20:14:35,075 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 73ad3832418d54490bc2efefb33a9d64:
Waiting for close lock at 1733948074642Running coprocessor pre-close hooks at 1733948074642Disabling compacts and flushes for region at 1733948074642Disabling writes for close at 1733948074642Obtaining lock to block concurrent updates at 1733948074643 (+1 ms)Preparing flush snapshotting stores in 73ad3832418d54490bc2efefb33a9d64 at 1733948074643Finished memstore snapshotting TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733948024278.73ad3832418d54490bc2efefb33a9d64., syncing WAL and waiting on mvcc, flushsize=dataSize=1076, getHeapSize=1392, getOffHeapSize=0, getCellsCount=1 at 1733948074643Flushing stores of TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733948024278.73ad3832418d54490bc2efefb33a9d64. at 1733948074644 (+1 ms)Flushing 73ad3832418d54490bc2efefb33a9d64/info: creating writer at 1733948074644Flushing 73ad3832418d54490bc2efefb33a9d64/info: appending metadata at 1733948074649 (+5 ms)Flushing 73ad3832418d54490bc2efefb33a9d64/info: closing flushed file at 1733948074649Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3d2c6dcf: reopening flushed file at 1733948075060 (+411 ms)Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 73ad3832418d54490bc2efefb33a9d64 in 425ms, sequenceid=22, compaction requested=true at 1733948075067 (+7 ms)Writing region close event to WAL at 1733948075071 (+4 ms)Running coprocessor post-close hooks at 1733948075075 (+4 ms)Closed at 1733948075075
2024-12-11T20:14:35,075 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733948024278.73ad3832418d54490bc2efefb33a9d64.
2024-12-11T20:14:35,243 INFO  [RS:0;9fe0640122ec:34019 {}] regionserver.HRegionServer(976): stopping server 9fe0640122ec,34019,1733948023445; all regions closed.
2024-12-11T20:14:35,244 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:14:35,244 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:14:35,244 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:14:35,244 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:14:35,244 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:14:35,246 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43661 is added to blk_1073741834_1010 (size=3306)
2024-12-11T20:14:35,247 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38159 is added to blk_1073741834_1010 (size=3306)
2024-12-11T20:14:35,249 DEBUG [RS:0;9fe0640122ec:34019 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/oldWALs
2024-12-11T20:14:35,249 INFO  [RS:0;9fe0640122ec:34019 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 9fe0640122ec%2C34019%2C1733948023445.meta:.meta(num 1733948024198)
2024-12-11T20:14:35,249 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:14:35,249 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:14:35,250 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:14:35,250 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:14:35,250 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:14:35,251 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38159 is added to blk_1073741844_1020 (size=1252)
2024-12-11T20:14:35,252 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43661 is added to blk_1073741844_1020 (size=1252)
2024-12-11T20:14:35,254 DEBUG [RS:0;9fe0640122ec:34019 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/oldWALs
2024-12-11T20:14:35,254 INFO  [RS:0;9fe0640122ec:34019 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 9fe0640122ec%2C34019%2C1733948023445:(num 1733948074630)
2024-12-11T20:14:35,255 DEBUG [RS:0;9fe0640122ec:34019 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T20:14:35,255 INFO  [RS:0;9fe0640122ec:34019 {}] regionserver.LeaseManager(133): Closed leases
2024-12-11T20:14:35,255 INFO  [RS:0;9fe0640122ec:34019 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-11T20:14:35,255 INFO  [RS:0;9fe0640122ec:34019 {}] hbase.ChoreService(370): Chore service for: regionserver/9fe0640122ec:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-12-11T20:14:35,255 INFO  [RS:0;9fe0640122ec:34019 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-11T20:14:35,255 INFO  [regionserver/9fe0640122ec:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-11T20:14:35,255 INFO  [RS:0;9fe0640122ec:34019 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34019
2024-12-11T20:14:35,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34019-0x100caf934d80001, quorum=127.0.0.1:63467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9fe0640122ec,34019,1733948023445
2024-12-11T20:14:35,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35237-0x100caf934d80000, quorum=127.0.0.1:63467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-11T20:14:35,258 INFO  [RS:0;9fe0640122ec:34019 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-11T20:14:35,259 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9fe0640122ec,34019,1733948023445]
2024-12-11T20:14:35,261 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9fe0640122ec,34019,1733948023445 already deleted, retry=false
2024-12-11T20:14:35,261 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 9fe0640122ec,34019,1733948023445 expired; onlineServers=0
2024-12-11T20:14:35,261 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '9fe0640122ec,35237,1733948023402' *****
2024-12-11T20:14:35,261 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-11T20:14:35,261 INFO  [M:0;9fe0640122ec:35237 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-11T20:14:35,261 INFO  [M:0;9fe0640122ec:35237 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-11T20:14:35,261 DEBUG [M:0;9fe0640122ec:35237 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-11T20:14:35,261 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-11T20:14:35,261 DEBUG [M:0;9fe0640122ec:35237 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-11T20:14:35,261 DEBUG [master/9fe0640122ec:0:becomeActiveMaster-HFileCleaner.large.0-1733948023600 {}] cleaner.HFileCleaner(306): Exit Thread[master/9fe0640122ec:0:becomeActiveMaster-HFileCleaner.large.0-1733948023600,5,FailOnTimeoutGroup]
2024-12-11T20:14:35,261 DEBUG [master/9fe0640122ec:0:becomeActiveMaster-HFileCleaner.small.0-1733948023600 {}] cleaner.HFileCleaner(306): Exit Thread[master/9fe0640122ec:0:becomeActiveMaster-HFileCleaner.small.0-1733948023600,5,FailOnTimeoutGroup]
2024-12-11T20:14:35,261 INFO  [M:0;9fe0640122ec:35237 {}] hbase.ChoreService(370): Chore service for: master/9fe0640122ec:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-12-11T20:14:35,261 INFO  [M:0;9fe0640122ec:35237 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-11T20:14:35,262 DEBUG [M:0;9fe0640122ec:35237 {}] master.HMaster(1795): Stopping service threads
2024-12-11T20:14:35,262 INFO  [M:0;9fe0640122ec:35237 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-11T20:14:35,262 INFO  [M:0;9fe0640122ec:35237 {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-11T20:14:35,262 INFO  [M:0;9fe0640122ec:35237 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-11T20:14:35,262 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-11T20:14:35,263 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35237-0x100caf934d80000, quorum=127.0.0.1:63467, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-11T20:14:35,263 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35237-0x100caf934d80000, quorum=127.0.0.1:63467, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:14:35,263 DEBUG [M:0;9fe0640122ec:35237 {}] zookeeper.ZKUtil(347): master:35237-0x100caf934d80000, quorum=127.0.0.1:63467, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-11T20:14:35,263 WARN  [M:0;9fe0640122ec:35237 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-11T20:14:35,263 INFO  [M:0;9fe0640122ec:35237 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/.lastflushedseqids
2024-12-11T20:14:35,268 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38159 is added to blk_1073741849_1025 (size=130)
2024-12-11T20:14:35,269 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43661 is added to blk_1073741849_1025 (size=130)
2024-12-11T20:14:35,269 INFO  [M:0;9fe0640122ec:35237 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-12-11T20:14:35,269 INFO  [M:0;9fe0640122ec:35237 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-11T20:14:35,269 DEBUG [M:0;9fe0640122ec:35237 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-11T20:14:35,269 INFO  [M:0;9fe0640122ec:35237 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T20:14:35,269 DEBUG [M:0;9fe0640122ec:35237 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T20:14:35,269 DEBUG [M:0;9fe0640122ec:35237 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-11T20:14:35,269 DEBUG [M:0;9fe0640122ec:35237 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T20:14:35,269 INFO  [M:0;9fe0640122ec:35237 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=43.59 KB heapSize=55 KB
2024-12-11T20:14:35,292 DEBUG [M:0;9fe0640122ec:35237 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5d1181b0fcb64cdfba6a9c6ddbfd8005 is 82, key is hbase:meta,,1/info:regioninfo/1733948024226/Put/seqid=0
2024-12-11T20:14:35,297 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43661 is added to blk_1073741850_1026 (size=5672)
2024-12-11T20:14:35,297 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38159 is added to blk_1073741850_1026 (size=5672)
2024-12-11T20:14:35,298 INFO  [M:0;9fe0640122ec:35237 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5d1181b0fcb64cdfba6a9c6ddbfd8005
2024-12-11T20:14:35,318 DEBUG [M:0;9fe0640122ec:35237 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a615afb3ddb74e128b2a17f5d09f594a is 799, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733948024649/Put/seqid=0
2024-12-11T20:14:35,321 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:35,321 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:35,322 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38159 is added to blk_1073741851_1027 (size=7823)
2024-12-11T20:14:35,323 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43661 is added to blk_1073741851_1027 (size=7823)
2024-12-11T20:14:35,323 INFO  [M:0;9fe0640122ec:35237 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.99 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a615afb3ddb74e128b2a17f5d09f594a
2024-12-11T20:14:35,328 INFO  [M:0;9fe0640122ec:35237 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a615afb3ddb74e128b2a17f5d09f594a
2024-12-11T20:14:35,343 DEBUG [M:0;9fe0640122ec:35237 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ece6a828ae8d45c2b783f208fd464ddd is 69, key is 9fe0640122ec,34019,1733948023445/rs:state/1733948023684/Put/seqid=0
2024-12-11T20:14:35,347 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38159 is added to blk_1073741852_1028 (size=5156)
2024-12-11T20:14:35,348 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43661 is added to blk_1073741852_1028 (size=5156)
2024-12-11T20:14:35,348 INFO  [M:0;9fe0640122ec:35237 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ece6a828ae8d45c2b783f208fd464ddd
2024-12-11T20:14:35,360 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34019-0x100caf934d80001, quorum=127.0.0.1:63467, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-11T20:14:35,360 INFO  [RS:0;9fe0640122ec:34019 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-11T20:14:35,360 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34019-0x100caf934d80001, quorum=127.0.0.1:63467, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-11T20:14:35,360 INFO  [RS:0;9fe0640122ec:34019 {}] regionserver.HRegionServer(1031): Exiting; stopping=9fe0640122ec,34019,1733948023445; zookeeper connection closed.
2024-12-11T20:14:35,360 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@20d361c8 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@20d361c8
2024-12-11T20:14:35,360 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-12-11T20:14:35,370 DEBUG [M:0;9fe0640122ec:35237 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/d0952679a38440b7ad85552c6e733bbd is 52, key is load_balancer_on/state:d/1733948024275/Put/seqid=0
2024-12-11T20:14:35,375 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43661 is added to blk_1073741853_1029 (size=5056)
2024-12-11T20:14:35,375 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38159 is added to blk_1073741853_1029 (size=5056)
2024-12-11T20:14:35,376 INFO  [M:0;9fe0640122ec:35237 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/d0952679a38440b7ad85552c6e733bbd
2024-12-11T20:14:35,382 DEBUG [M:0;9fe0640122ec:35237 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5d1181b0fcb64cdfba6a9c6ddbfd8005 as hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/5d1181b0fcb64cdfba6a9c6ddbfd8005
2024-12-11T20:14:35,387 INFO  [M:0;9fe0640122ec:35237 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/5d1181b0fcb64cdfba6a9c6ddbfd8005, entries=8, sequenceid=121, filesize=5.5 K
2024-12-11T20:14:35,387 DEBUG [M:0;9fe0640122ec:35237 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a615afb3ddb74e128b2a17f5d09f594a as hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a615afb3ddb74e128b2a17f5d09f594a
2024-12-11T20:14:35,392 INFO  [M:0;9fe0640122ec:35237 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a615afb3ddb74e128b2a17f5d09f594a
2024-12-11T20:14:35,392 INFO  [M:0;9fe0640122ec:35237 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a615afb3ddb74e128b2a17f5d09f594a, entries=14, sequenceid=121, filesize=7.6 K
2024-12-11T20:14:35,392 DEBUG [M:0;9fe0640122ec:35237 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ece6a828ae8d45c2b783f208fd464ddd as hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ece6a828ae8d45c2b783f208fd464ddd
2024-12-11T20:14:35,397 INFO  [M:0;9fe0640122ec:35237 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ece6a828ae8d45c2b783f208fd464ddd, entries=1, sequenceid=121, filesize=5.0 K
2024-12-11T20:14:35,397 DEBUG [M:0;9fe0640122ec:35237 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/d0952679a38440b7ad85552c6e733bbd as hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/d0952679a38440b7ad85552c6e733bbd
2024-12-11T20:14:35,402 INFO  [M:0;9fe0640122ec:35237 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/5ee8a25d-73d3-969a-176e-c6820aa0c72b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/d0952679a38440b7ad85552c6e733bbd, entries=1, sequenceid=121, filesize=4.9 K
2024-12-11T20:14:35,403 INFO  [M:0;9fe0640122ec:35237 {}] regionserver.HRegion(3140): Finished flush of dataSize ~43.59 KB/44641, heapSize ~54.94 KB/56256, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 134ms, sequenceid=121, compaction requested=false
2024-12-11T20:14:35,404 INFO  [M:0;9fe0640122ec:35237 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T20:14:35,405 DEBUG [M:0;9fe0640122ec:35237 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733948075269Disabling compacts and flushes for region at 1733948075269Disabling writes for close at 1733948075269Obtaining lock to block concurrent updates at 1733948075269Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733948075269Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=44641, getHeapSize=56256, getOffHeapSize=0, getCellsCount=140 at 1733948075270 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733948075271 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733948075271Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733948075292 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733948075292Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733948075303 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733948075317 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733948075317Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733948075328 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733948075342 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733948075342Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733948075353 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733948075369 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733948075369Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2b0720cd: reopening flushed file at 1733948075381 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7b13be23: reopening flushed file at 1733948075387 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@73052b12: reopening flushed file at 1733948075392 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@90d4354: reopening flushed file at 1733948075397 (+5 ms)Finished flush of dataSize ~43.59 KB/44641, heapSize ~54.94 KB/56256, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 134ms, sequenceid=121, compaction requested=false at 1733948075403 (+6 ms)Writing region close event to WAL at 1733948075404 (+1 ms)Closed at 1733948075404
2024-12-11T20:14:35,405 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:14:35,405 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:14:35,405 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:14:35,405 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:14:35,405 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:14:35,407 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38159 is added to blk_1073741830_1006 (size=53038)
2024-12-11T20:14:35,407 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43661 is added to blk_1073741830_1006 (size=53038)
2024-12-11T20:14:35,408 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-11T20:14:35,408 INFO  [M:0;9fe0640122ec:35237 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-12-11T20:14:35,408 INFO  [M:0;9fe0640122ec:35237 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35237
2024-12-11T20:14:35,408 INFO  [M:0;9fe0640122ec:35237 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-11T20:14:35,510 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35237-0x100caf934d80000, quorum=127.0.0.1:63467, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-11T20:14:35,511 INFO  [M:0;9fe0640122ec:35237 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-11T20:14:35,511 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35237-0x100caf934d80000, quorum=127.0.0.1:63467, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-11T20:14:35,513 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3d40a54d{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T20:14:35,513 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@74b5ebca{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-11T20:14:35,513 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-11T20:14:35,513 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@438a440e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-11T20:14:35,514 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@72ef9fa2{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a9019bd4-2e2e-eb94-0662-008269ec518f/hadoop.log.dir/,STOPPED}
2024-12-11T20:14:35,515 WARN  [BP-313382295-172.17.0.2-1733948022749 heartbeating to localhost/127.0.0.1:46041 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-11T20:14:35,515 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-11T20:14:35,515 WARN  [BP-313382295-172.17.0.2-1733948022749 heartbeating to localhost/127.0.0.1:46041 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-313382295-172.17.0.2-1733948022749 (Datanode Uuid 56bb8b2c-8d4a-4065-a135-d006da89b1ef) service to localhost/127.0.0.1:46041
2024-12-11T20:14:35,515 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-11T20:14:35,516 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a9019bd4-2e2e-eb94-0662-008269ec518f/cluster_e92aeb1d-de81-341f-1b6e-20867c1a221d/data/data3/current/BP-313382295-172.17.0.2-1733948022749 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T20:14:35,516 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a9019bd4-2e2e-eb94-0662-008269ec518f/cluster_e92aeb1d-de81-341f-1b6e-20867c1a221d/data/data4/current/BP-313382295-172.17.0.2-1733948022749 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T20:14:35,516 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-11T20:14:35,518 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5d8d7f9b{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T20:14:35,519 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@190ad9e7{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-11T20:14:35,519 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-11T20:14:35,519 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@16aeea80{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-11T20:14:35,519 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@b24fbcd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a9019bd4-2e2e-eb94-0662-008269ec518f/hadoop.log.dir/,STOPPED}
2024-12-11T20:14:35,520 WARN  [BP-313382295-172.17.0.2-1733948022749 heartbeating to localhost/127.0.0.1:46041 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-11T20:14:35,520 WARN  [BP-313382295-172.17.0.2-1733948022749 heartbeating to localhost/127.0.0.1:46041 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-313382295-172.17.0.2-1733948022749 (Datanode Uuid 1d9df031-89dd-472d-b7cd-4182e4049549) service to localhost/127.0.0.1:46041
2024-12-11T20:14:35,520 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-11T20:14:35,520 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-11T20:14:35,521 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a9019bd4-2e2e-eb94-0662-008269ec518f/cluster_e92aeb1d-de81-341f-1b6e-20867c1a221d/data/data1/current/BP-313382295-172.17.0.2-1733948022749 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T20:14:35,521 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a9019bd4-2e2e-eb94-0662-008269ec518f/cluster_e92aeb1d-de81-341f-1b6e-20867c1a221d/data/data2/current/BP-313382295-172.17.0.2-1733948022749 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T20:14:35,521 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-11T20:14:35,528 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@21edc7fd{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-11T20:14:35,528 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7b612817{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-11T20:14:35,528 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-11T20:14:35,528 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@29b40997{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-11T20:14:35,528 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@c464107{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a9019bd4-2e2e-eb94-0662-008269ec518f/hadoop.log.dir/,STOPPED}
2024-12-11T20:14:35,534 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-12-11T20:14:35,552 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-12-11T20:14:35,561 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=208 (was 179)
Potentially hanging thread: nioEventLoopGroup-36-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-13
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-34-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:46041
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-37-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-12-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-36-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:46041 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:46041 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-35-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-12-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-36-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-35-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-35-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-34-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-12
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-14
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-37-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-12-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:46041 from jenkins.hfs.5
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-37-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:46041
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-34-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:46041
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.5@localhost:46041
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: regionserver/9fe0640122ec:0.leaseChecker
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.regionserver.LeaseManager.run(LeaseManager.java:82)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:46041
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=483 (was 457) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=72 (was 138), ProcessCount=11 (was 11), AvailableMemoryMB=316 (was 293) - AvailableMemoryMB LEAK? -
2024-12-11T20:14:35,568 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRolling Thread=208, OpenFileDescriptor=483, MaxFileDescriptor=1048576, SystemLoadAverage=72, ProcessCount=11, AvailableMemoryMB=316
2024-12-11T20:14:35,568 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-11T20:14:35,569 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a9019bd4-2e2e-eb94-0662-008269ec518f/hadoop.log.dir so I do NOT create it in target/test-data/c90893ce-dd95-6d20-b72c-9d6839befe15
2024-12-11T20:14:35,569 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a9019bd4-2e2e-eb94-0662-008269ec518f/hadoop.tmp.dir so I do NOT create it in target/test-data/c90893ce-dd95-6d20-b72c-9d6839befe15
2024-12-11T20:14:35,569 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90893ce-dd95-6d20-b72c-9d6839befe15/cluster_7587c867-25df-d69c-d06b-fd5497de92d4, deleteOnExit=true
2024-12-11T20:14:35,569 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-12-11T20:14:35,569 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90893ce-dd95-6d20-b72c-9d6839befe15/test.cache.data in system properties and HBase conf
2024-12-11T20:14:35,569 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90893ce-dd95-6d20-b72c-9d6839befe15/hadoop.tmp.dir in system properties and HBase conf
2024-12-11T20:14:35,569 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90893ce-dd95-6d20-b72c-9d6839befe15/hadoop.log.dir in system properties and HBase conf
2024-12-11T20:14:35,569 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90893ce-dd95-6d20-b72c-9d6839befe15/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-11T20:14:35,569 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90893ce-dd95-6d20-b72c-9d6839befe15/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-11T20:14:35,569 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-12-11T20:14:35,569 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-11T20:14:35,569 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90893ce-dd95-6d20-b72c-9d6839befe15/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-11T20:14:35,569 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90893ce-dd95-6d20-b72c-9d6839befe15/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-11T20:14:35,569 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90893ce-dd95-6d20-b72c-9d6839befe15/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-11T20:14:35,570 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90893ce-dd95-6d20-b72c-9d6839befe15/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-11T20:14:35,570 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90893ce-dd95-6d20-b72c-9d6839befe15/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-11T20:14:35,570 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90893ce-dd95-6d20-b72c-9d6839befe15/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-11T20:14:35,570 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90893ce-dd95-6d20-b72c-9d6839befe15/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-11T20:14:35,570 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90893ce-dd95-6d20-b72c-9d6839befe15/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-11T20:14:35,570 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90893ce-dd95-6d20-b72c-9d6839befe15/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-11T20:14:35,570 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90893ce-dd95-6d20-b72c-9d6839befe15/nfs.dump.dir in system properties and HBase conf
2024-12-11T20:14:35,570 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90893ce-dd95-6d20-b72c-9d6839befe15/java.io.tmpdir in system properties and HBase conf
2024-12-11T20:14:35,570 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90893ce-dd95-6d20-b72c-9d6839befe15/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-11T20:14:35,570 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90893ce-dd95-6d20-b72c-9d6839befe15/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-11T20:14:35,570 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90893ce-dd95-6d20-b72c-9d6839befe15/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-11T20:14:35,583 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-11T20:14:35,649 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T20:14:35,652 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-11T20:14:35,653 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-11T20:14:35,653 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-11T20:14:35,653 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-11T20:14:35,654 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T20:14:35,654 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2f05aefd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90893ce-dd95-6d20-b72c-9d6839befe15/hadoop.log.dir/,AVAILABLE}
2024-12-11T20:14:35,655 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@40a5a9d2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-11T20:14:35,698 INFO  [regionserver/9fe0640122ec:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-11T20:14:35,770 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@10e59f6e{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90893ce-dd95-6d20-b72c-9d6839befe15/java.io.tmpdir/jetty-localhost-38807-hadoop-hdfs-3_4_1-tests_jar-_-any-9598954884926238817/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-11T20:14:35,771 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@56d12e32{HTTP/1.1, (http/1.1)}{localhost:38807}
2024-12-11T20:14:35,771 INFO  [Time-limited test {}] server.Server(415): Started @239366ms
2024-12-11T20:14:35,784 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-11T20:14:35,835 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T20:14:35,838 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-11T20:14:35,838 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-11T20:14:35,838 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-11T20:14:35,838 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-11T20:14:35,839 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@444d0b71{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90893ce-dd95-6d20-b72c-9d6839befe15/hadoop.log.dir/,AVAILABLE}
2024-12-11T20:14:35,839 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7193a060{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-11T20:14:35,953 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@76385d85{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90893ce-dd95-6d20-b72c-9d6839befe15/java.io.tmpdir/jetty-localhost-39135-hadoop-hdfs-3_4_1-tests_jar-_-any-43607094769510628/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T20:14:35,954 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@284a8a01{HTTP/1.1, (http/1.1)}{localhost:39135}
2024-12-11T20:14:35,954 INFO  [Time-limited test {}] server.Server(415): Started @239549ms
2024-12-11T20:14:35,975 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-11T20:14:36,005 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T20:14:36,008 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-11T20:14:36,008 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-11T20:14:36,008 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-11T20:14:36,008 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-11T20:14:36,009 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@18d1ee92{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90893ce-dd95-6d20-b72c-9d6839befe15/hadoop.log.dir/,AVAILABLE}
2024-12-11T20:14:36,009 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7cd1acca{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-11T20:14:36,072 WARN  [Thread-1960 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90893ce-dd95-6d20-b72c-9d6839befe15/cluster_7587c867-25df-d69c-d06b-fd5497de92d4/data/data2/current/BP-1861006598-172.17.0.2-1733948075590/current, will proceed with Du for space computation calculation, 
2024-12-11T20:14:36,072 WARN  [Thread-1959 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90893ce-dd95-6d20-b72c-9d6839befe15/cluster_7587c867-25df-d69c-d06b-fd5497de92d4/data/data1/current/BP-1861006598-172.17.0.2-1733948075590/current, will proceed with Du for space computation calculation, 
2024-12-11T20:14:36,095 WARN  [Thread-1938 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-11T20:14:36,097 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x44e1f8e9d4e3e3c8 with lease ID 0x41293715202bd0a5: Processing first storage report for DS-8dea2e60-1619-4aea-8889-61241c3965d0 from datanode DatanodeRegistration(127.0.0.1:33119, datanodeUuid=5bcb4a8d-8e7a-44b6-8f9a-f9e4d482fcf2, infoPort=33307, infoSecurePort=0, ipcPort=43677, storageInfo=lv=-57;cid=testClusterID;nsid=864646626;c=1733948075590)
2024-12-11T20:14:36,097 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x44e1f8e9d4e3e3c8 with lease ID 0x41293715202bd0a5: from storage DS-8dea2e60-1619-4aea-8889-61241c3965d0 node DatanodeRegistration(127.0.0.1:33119, datanodeUuid=5bcb4a8d-8e7a-44b6-8f9a-f9e4d482fcf2, infoPort=33307, infoSecurePort=0, ipcPort=43677, storageInfo=lv=-57;cid=testClusterID;nsid=864646626;c=1733948075590), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T20:14:36,097 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x44e1f8e9d4e3e3c8 with lease ID 0x41293715202bd0a5: Processing first storage report for DS-b6ee3d04-86ca-4a3b-91d0-5b60062c0824 from datanode DatanodeRegistration(127.0.0.1:33119, datanodeUuid=5bcb4a8d-8e7a-44b6-8f9a-f9e4d482fcf2, infoPort=33307, infoSecurePort=0, ipcPort=43677, storageInfo=lv=-57;cid=testClusterID;nsid=864646626;c=1733948075590)
2024-12-11T20:14:36,097 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x44e1f8e9d4e3e3c8 with lease ID 0x41293715202bd0a5: from storage DS-b6ee3d04-86ca-4a3b-91d0-5b60062c0824 node DatanodeRegistration(127.0.0.1:33119, datanodeUuid=5bcb4a8d-8e7a-44b6-8f9a-f9e4d482fcf2, infoPort=33307, infoSecurePort=0, ipcPort=43677, storageInfo=lv=-57;cid=testClusterID;nsid=864646626;c=1733948075590), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T20:14:36,141 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@601b78f7{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90893ce-dd95-6d20-b72c-9d6839befe15/java.io.tmpdir/jetty-localhost-40529-hadoop-hdfs-3_4_1-tests_jar-_-any-7177261998846595450/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T20:14:36,142 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@35a5806e{HTTP/1.1, (http/1.1)}{localhost:40529}
2024-12-11T20:14:36,142 INFO  [Time-limited test {}] server.Server(415): Started @239738ms
2024-12-11T20:14:36,143 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-11T20:14:36,223 WARN  [Thread-1985 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90893ce-dd95-6d20-b72c-9d6839befe15/cluster_7587c867-25df-d69c-d06b-fd5497de92d4/data/data3/current/BP-1861006598-172.17.0.2-1733948075590/current, will proceed with Du for space computation calculation, 
2024-12-11T20:14:36,223 WARN  [Thread-1986 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90893ce-dd95-6d20-b72c-9d6839befe15/cluster_7587c867-25df-d69c-d06b-fd5497de92d4/data/data4/current/BP-1861006598-172.17.0.2-1733948075590/current, will proceed with Du for space computation calculation, 
2024-12-11T20:14:36,243 WARN  [Thread-1974 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-11T20:14:36,246 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x189971c76d16422d with lease ID 0x41293715202bd0a6: Processing first storage report for DS-e19c7f55-91a8-49f8-9392-1ddcf0c50df1 from datanode DatanodeRegistration(127.0.0.1:35725, datanodeUuid=21266db9-4349-4f71-97ad-93cb37c31e02, infoPort=39125, infoSecurePort=0, ipcPort=33377, storageInfo=lv=-57;cid=testClusterID;nsid=864646626;c=1733948075590)
2024-12-11T20:14:36,246 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x189971c76d16422d with lease ID 0x41293715202bd0a6: from storage DS-e19c7f55-91a8-49f8-9392-1ddcf0c50df1 node DatanodeRegistration(127.0.0.1:35725, datanodeUuid=21266db9-4349-4f71-97ad-93cb37c31e02, infoPort=39125, infoSecurePort=0, ipcPort=33377, storageInfo=lv=-57;cid=testClusterID;nsid=864646626;c=1733948075590), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T20:14:36,246 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x189971c76d16422d with lease ID 0x41293715202bd0a6: Processing first storage report for DS-a64bdc95-23f9-488a-893e-9d7c9eebf5d0 from datanode DatanodeRegistration(127.0.0.1:35725, datanodeUuid=21266db9-4349-4f71-97ad-93cb37c31e02, infoPort=39125, infoSecurePort=0, ipcPort=33377, storageInfo=lv=-57;cid=testClusterID;nsid=864646626;c=1733948075590)
2024-12-11T20:14:36,246 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x189971c76d16422d with lease ID 0x41293715202bd0a6: from storage DS-a64bdc95-23f9-488a-893e-9d7c9eebf5d0 node DatanodeRegistration(127.0.0.1:35725, datanodeUuid=21266db9-4349-4f71-97ad-93cb37c31e02, infoPort=39125, infoSecurePort=0, ipcPort=33377, storageInfo=lv=-57;cid=testClusterID;nsid=864646626;c=1733948075590), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T20:14:36,265 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90893ce-dd95-6d20-b72c-9d6839befe15
2024-12-11T20:14:36,268 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90893ce-dd95-6d20-b72c-9d6839befe15/cluster_7587c867-25df-d69c-d06b-fd5497de92d4/zookeeper_0, clientPort=53734, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90893ce-dd95-6d20-b72c-9d6839befe15/cluster_7587c867-25df-d69c-d06b-fd5497de92d4/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90893ce-dd95-6d20-b72c-9d6839befe15/cluster_7587c867-25df-d69c-d06b-fd5497de92d4/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-11T20:14:36,269 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=53734
2024-12-11T20:14:36,269 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T20:14:36,270 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T20:14:36,278 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35725 is added to blk_1073741825_1001 (size=7)
2024-12-11T20:14:36,279 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741825_1001 (size=7)
2024-12-11T20:14:36,279 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c with version=8
2024-12-11T20:14:36,280 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/hbase-staging
2024-12-11T20:14:36,282 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/9fe0640122ec:0 server-side Connection retries=45
2024-12-11T20:14:36,282 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-11T20:14:36,282 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-11T20:14:36,282 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-11T20:14:36,282 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-11T20:14:36,282 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-11T20:14:36,282 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-12-11T20:14:36,282 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-11T20:14:36,283 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46569
2024-12-11T20:14:36,284 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:46569 connecting to ZooKeeper ensemble=127.0.0.1:53734
2024-12-11T20:14:36,289 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:465690x0, quorum=127.0.0.1:53734, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-11T20:14:36,290 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:46569-0x100cafa036a0000 connected
2024-12-11T20:14:36,302 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T20:14:36,303 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T20:14:36,305 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46569-0x100cafa036a0000, quorum=127.0.0.1:53734, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-11T20:14:36,305 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c, hbase.cluster.distributed=false
2024-12-11T20:14:36,306 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46569-0x100cafa036a0000, quorum=127.0.0.1:53734, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-11T20:14:36,307 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46569
2024-12-11T20:14:36,307 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46569
2024-12-11T20:14:36,307 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46569
2024-12-11T20:14:36,307 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46569
2024-12-11T20:14:36,308 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46569
2024-12-11T20:14:36,322 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:36,322 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:36,323 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/9fe0640122ec:0 server-side Connection retries=45
2024-12-11T20:14:36,323 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-11T20:14:36,323 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-11T20:14:36,323 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-11T20:14:36,323 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-11T20:14:36,323 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-11T20:14:36,323 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-11T20:14:36,323 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-11T20:14:36,324 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:32831
2024-12-11T20:14:36,325 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:32831 connecting to ZooKeeper ensemble=127.0.0.1:53734
2024-12-11T20:14:36,326 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T20:14:36,327 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T20:14:36,331 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:328310x0, quorum=127.0.0.1:53734, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-11T20:14:36,331 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:328310x0, quorum=127.0.0.1:53734, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-11T20:14:36,331 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:32831-0x100cafa036a0001 connected
2024-12-11T20:14:36,332 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-11T20:14:36,335 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-11T20:14:36,336 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:32831-0x100cafa036a0001, quorum=127.0.0.1:53734, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-11T20:14:36,337 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:32831-0x100cafa036a0001, quorum=127.0.0.1:53734, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-11T20:14:36,337 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=32831
2024-12-11T20:14:36,338 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=32831
2024-12-11T20:14:36,338 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=32831
2024-12-11T20:14:36,339 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=32831
2024-12-11T20:14:36,339 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=32831
2024-12-11T20:14:36,352 DEBUG [M:0;9fe0640122ec:46569 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;9fe0640122ec:46569
2024-12-11T20:14:36,353 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/9fe0640122ec,46569,1733948076281
2024-12-11T20:14:36,354 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46569-0x100cafa036a0000, quorum=127.0.0.1:53734, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-11T20:14:36,354 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32831-0x100cafa036a0001, quorum=127.0.0.1:53734, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-11T20:14:36,355 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46569-0x100cafa036a0000, quorum=127.0.0.1:53734, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/9fe0640122ec,46569,1733948076281
2024-12-11T20:14:36,356 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32831-0x100cafa036a0001, quorum=127.0.0.1:53734, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-11T20:14:36,356 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46569-0x100cafa036a0000, quorum=127.0.0.1:53734, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:14:36,356 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32831-0x100cafa036a0001, quorum=127.0.0.1:53734, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:14:36,357 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46569-0x100cafa036a0000, quorum=127.0.0.1:53734, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-11T20:14:36,357 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/9fe0640122ec,46569,1733948076281 from backup master directory
2024-12-11T20:14:36,358 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46569-0x100cafa036a0000, quorum=127.0.0.1:53734, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/9fe0640122ec,46569,1733948076281
2024-12-11T20:14:36,358 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32831-0x100cafa036a0001, quorum=127.0.0.1:53734, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-11T20:14:36,358 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46569-0x100cafa036a0000, quorum=127.0.0.1:53734, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-11T20:14:36,358 WARN  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-11T20:14:36,358 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=9fe0640122ec,46569,1733948076281
2024-12-11T20:14:36,362 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/hbase.id] with ID: b56e9c7a-9ff4-4371-9f7e-048538138346
2024-12-11T20:14:36,362 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/.tmp/hbase.id
2024-12-11T20:14:36,367 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741826_1002 (size=42)
2024-12-11T20:14:36,368 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35725 is added to blk_1073741826_1002 (size=42)
2024-12-11T20:14:36,368 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/.tmp/hbase.id]:[hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/hbase.id]
2024-12-11T20:14:36,378 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T20:14:36,378 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-12-11T20:14:36,379 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-12-11T20:14:36,381 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46569-0x100cafa036a0000, quorum=127.0.0.1:53734, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:14:36,381 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32831-0x100cafa036a0001, quorum=127.0.0.1:53734, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:14:36,387 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35725 is added to blk_1073741827_1003 (size=196)
2024-12-11T20:14:36,387 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741827_1003 (size=196)
2024-12-11T20:14:36,388 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-11T20:14:36,388 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-11T20:14:36,389 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-11T20:14:36,396 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741828_1004 (size=1189)
2024-12-11T20:14:36,397 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35725 is added to blk_1073741828_1004 (size=1189)
2024-12-11T20:14:36,397 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/MasterData/data/master/store
2024-12-11T20:14:36,403 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35725 is added to blk_1073741829_1005 (size=34)
2024-12-11T20:14:36,403 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741829_1005 (size=34)
2024-12-11T20:14:36,404 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T20:14:36,404 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-11T20:14:36,404 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T20:14:36,404 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T20:14:36,404 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-11T20:14:36,404 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T20:14:36,404 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T20:14:36,404 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733948076404Disabling compacts and flushes for region at 1733948076404Disabling writes for close at 1733948076404Writing region close event to WAL at 1733948076404Closed at 1733948076404
2024-12-11T20:14:36,405 WARN  [master/9fe0640122ec:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/MasterData/data/master/store/.initializing
2024-12-11T20:14:36,405 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/MasterData/WALs/9fe0640122ec,46569,1733948076281
2024-12-11T20:14:36,407 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9fe0640122ec%2C46569%2C1733948076281, suffix=, logDir=hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/MasterData/WALs/9fe0640122ec,46569,1733948076281, archiveDir=hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/MasterData/oldWALs, maxLogs=10
2024-12-11T20:14:36,408 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9fe0640122ec%2C46569%2C1733948076281.1733948076408
2024-12-11T20:14:36,412 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/MasterData/WALs/9fe0640122ec,46569,1733948076281/9fe0640122ec%2C46569%2C1733948076281.1733948076408
2024-12-11T20:14:36,413 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33307:33307),(127.0.0.1/127.0.0.1:39125:39125)]
2024-12-11T20:14:36,413 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-11T20:14:36,414 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T20:14:36,414 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T20:14:36,414 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T20:14:36,415 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-11T20:14:36,416 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-11T20:14:36,416 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:14:36,416 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T20:14:36,416 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-11T20:14:36,417 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-11T20:14:36,417 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:14:36,418 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-11T20:14:36,418 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-11T20:14:36,419 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-11T20:14:36,419 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:14:36,419 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-11T20:14:36,419 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-11T20:14:36,420 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-11T20:14:36,420 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:14:36,420 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-11T20:14:36,421 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T20:14:36,421 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-11T20:14:36,422 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-11T20:14:36,423 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T20:14:36,423 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T20:14:36,423 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-11T20:14:36,424 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T20:14:36,426 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-11T20:14:36,426 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=802620, jitterRate=0.020584434270858765}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-11T20:14:36,427 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1733948076414Initializing all the Stores at 1733948076414Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733948076414Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733948076415 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733948076415Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733948076415Cleaning up temporary data from old regions at 1733948076423 (+8 ms)Region opened successfully at 1733948076427 (+4 ms)
2024-12-11T20:14:36,427 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-11T20:14:36,430 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43bf0184, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9fe0640122ec/172.17.0.2:0
2024-12-11T20:14:36,431 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-12-11T20:14:36,431 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-11T20:14:36,431 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-11T20:14:36,431 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-11T20:14:36,431 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-12-11T20:14:36,432 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-12-11T20:14:36,432 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-11T20:14:36,434 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-11T20:14:36,434 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46569-0x100cafa036a0000, quorum=127.0.0.1:53734, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-11T20:14:36,435 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-12-11T20:14:36,436 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-11T20:14:36,436 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46569-0x100cafa036a0000, quorum=127.0.0.1:53734, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-11T20:14:36,437 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-12-11T20:14:36,437 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-11T20:14:36,438 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46569-0x100cafa036a0000, quorum=127.0.0.1:53734, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-11T20:14:36,439 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-12-11T20:14:36,440 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46569-0x100cafa036a0000, quorum=127.0.0.1:53734, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-11T20:14:36,441 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-12-11T20:14:36,442 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46569-0x100cafa036a0000, quorum=127.0.0.1:53734, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-11T20:14:36,443 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-11T20:14:36,445 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32831-0x100cafa036a0001, quorum=127.0.0.1:53734, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-11T20:14:36,445 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46569-0x100cafa036a0000, quorum=127.0.0.1:53734, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-11T20:14:36,445 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32831-0x100cafa036a0001, quorum=127.0.0.1:53734, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:14:36,445 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46569-0x100cafa036a0000, quorum=127.0.0.1:53734, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:14:36,445 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=9fe0640122ec,46569,1733948076281, sessionid=0x100cafa036a0000, setting cluster-up flag (Was=false)
2024-12-11T20:14:36,447 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32831-0x100cafa036a0001, quorum=127.0.0.1:53734, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:14:36,447 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46569-0x100cafa036a0000, quorum=127.0.0.1:53734, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:14:36,451 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-11T20:14:36,452 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9fe0640122ec,46569,1733948076281
2024-12-11T20:14:36,455 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46569-0x100cafa036a0000, quorum=127.0.0.1:53734, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:14:36,455 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32831-0x100cafa036a0001, quorum=127.0.0.1:53734, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:14:36,459 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-11T20:14:36,460 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9fe0640122ec,46569,1733948076281
2024-12-11T20:14:36,461 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-12-11T20:14:36,462 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-12-11T20:14:36,463 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-12-11T20:14:36,463 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-11T20:14:36,463 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 9fe0640122ec,46569,1733948076281
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-11T20:14:36,464 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/9fe0640122ec:0, corePoolSize=5, maxPoolSize=5
2024-12-11T20:14:36,464 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/9fe0640122ec:0, corePoolSize=5, maxPoolSize=5
2024-12-11T20:14:36,464 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/9fe0640122ec:0, corePoolSize=5, maxPoolSize=5
2024-12-11T20:14:36,464 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/9fe0640122ec:0, corePoolSize=5, maxPoolSize=5
2024-12-11T20:14:36,464 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/9fe0640122ec:0, corePoolSize=10, maxPoolSize=10
2024-12-11T20:14:36,464 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:14:36,464 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/9fe0640122ec:0, corePoolSize=2, maxPoolSize=2
2024-12-11T20:14:36,464 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:14:36,466 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-11T20:14:36,466 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733948106466
2024-12-11T20:14:36,466 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-12-11T20:14:36,466 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-11T20:14:36,466 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-11T20:14:36,466 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-11T20:14:36,466 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-11T20:14:36,466 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-11T20:14:36,466 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-11T20:14:36,466 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-11T20:14:36,467 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-11T20:14:36,467 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-11T20:14:36,467 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-11T20:14:36,467 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:14:36,467 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-11T20:14:36,467 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-11T20:14:36,467 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-11T20:14:36,467 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/9fe0640122ec:0:becomeActiveMaster-HFileCleaner.large.0-1733948076467,5,FailOnTimeoutGroup]
2024-12-11T20:14:36,467 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/9fe0640122ec:0:becomeActiveMaster-HFileCleaner.small.0-1733948076467,5,FailOnTimeoutGroup]
2024-12-11T20:14:36,467 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-11T20:14:36,467 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-11T20:14:36,468 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-11T20:14:36,468 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-11T20:14:36,473 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741831_1007 (size=1321)
2024-12-11T20:14:36,473 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35725 is added to blk_1073741831_1007 (size=1321)
2024-12-11T20:14:36,474 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-12-11T20:14:36,474 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c
2024-12-11T20:14:36,479 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741832_1008 (size=32)
2024-12-11T20:14:36,480 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35725 is added to blk_1073741832_1008 (size=32)
2024-12-11T20:14:36,480 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T20:14:36,481 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-11T20:14:36,482 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-11T20:14:36,482 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:14:36,483 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T20:14:36,483 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-11T20:14:36,484 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-11T20:14:36,484 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:14:36,484 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T20:14:36,484 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-11T20:14:36,485 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-11T20:14:36,485 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:14:36,486 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T20:14:36,486 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-11T20:14:36,487 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-11T20:14:36,487 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:14:36,487 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T20:14:36,487 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-11T20:14:36,488 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/hbase/meta/1588230740
2024-12-11T20:14:36,488 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/hbase/meta/1588230740
2024-12-11T20:14:36,489 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-11T20:14:36,489 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-11T20:14:36,490 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-11T20:14:36,491 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-11T20:14:36,492 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-11T20:14:36,493 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=795428, jitterRate=0.011439070105552673}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-11T20:14:36,493 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1733948076480Initializing all the Stores at 1733948076481 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733948076481Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733948076481Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733948076481Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733948076481Cleaning up temporary data from old regions at 1733948076489 (+8 ms)Region opened successfully at 1733948076493 (+4 ms)
2024-12-11T20:14:36,493 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-11T20:14:36,493 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-11T20:14:36,493 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-11T20:14:36,494 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-11T20:14:36,494 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-11T20:14:36,494 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-11T20:14:36,494 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733948076493Disabling compacts and flushes for region at 1733948076493Disabling writes for close at 1733948076494 (+1 ms)Writing region close event to WAL at 1733948076494Closed at 1733948076494
2024-12-11T20:14:36,495 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-11T20:14:36,495 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-12-11T20:14:36,495 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-11T20:14:36,497 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-11T20:14:36,498 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-11T20:14:36,542 INFO  [RS:0;9fe0640122ec:32831 {}] regionserver.HRegionServer(746): ClusterId : b56e9c7a-9ff4-4371-9f7e-048538138346
2024-12-11T20:14:36,542 DEBUG [RS:0;9fe0640122ec:32831 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-11T20:14:36,544 DEBUG [RS:0;9fe0640122ec:32831 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-11T20:14:36,544 DEBUG [RS:0;9fe0640122ec:32831 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-11T20:14:36,546 DEBUG [RS:0;9fe0640122ec:32831 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-11T20:14:36,546 DEBUG [RS:0;9fe0640122ec:32831 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@647c0e8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9fe0640122ec/172.17.0.2:0
2024-12-11T20:14:36,558 DEBUG [RS:0;9fe0640122ec:32831 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;9fe0640122ec:32831
2024-12-11T20:14:36,558 INFO  [RS:0;9fe0640122ec:32831 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-11T20:14:36,558 INFO  [RS:0;9fe0640122ec:32831 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-11T20:14:36,558 DEBUG [RS:0;9fe0640122ec:32831 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-11T20:14:36,559 INFO  [RS:0;9fe0640122ec:32831 {}] regionserver.HRegionServer(2659): reportForDuty to master=9fe0640122ec,46569,1733948076281 with port=32831, startcode=1733948076323
2024-12-11T20:14:36,559 DEBUG [RS:0;9fe0640122ec:32831 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-11T20:14:36,561 INFO  [HMaster-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39933, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.6 (auth:SIMPLE), service=RegionServerStatusService
2024-12-11T20:14:36,562 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46569 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9fe0640122ec,32831,1733948076323
2024-12-11T20:14:36,562 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46569 {}] master.ServerManager(517): Registering regionserver=9fe0640122ec,32831,1733948076323
2024-12-11T20:14:36,564 DEBUG [RS:0;9fe0640122ec:32831 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c
2024-12-11T20:14:36,564 DEBUG [RS:0;9fe0640122ec:32831 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:37589
2024-12-11T20:14:36,564 DEBUG [RS:0;9fe0640122ec:32831 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-11T20:14:36,566 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46569-0x100cafa036a0000, quorum=127.0.0.1:53734, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-11T20:14:36,566 DEBUG [RS:0;9fe0640122ec:32831 {}] zookeeper.ZKUtil(111): regionserver:32831-0x100cafa036a0001, quorum=127.0.0.1:53734, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9fe0640122ec,32831,1733948076323
2024-12-11T20:14:36,566 WARN  [RS:0;9fe0640122ec:32831 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-11T20:14:36,566 INFO  [RS:0;9fe0640122ec:32831 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-11T20:14:36,566 DEBUG [RS:0;9fe0640122ec:32831 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/WALs/9fe0640122ec,32831,1733948076323
2024-12-11T20:14:36,566 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9fe0640122ec,32831,1733948076323]
2024-12-11T20:14:36,569 INFO  [RS:0;9fe0640122ec:32831 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-11T20:14:36,571 INFO  [RS:0;9fe0640122ec:32831 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-11T20:14:36,572 INFO  [RS:0;9fe0640122ec:32831 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-11T20:14:36,572 INFO  [RS:0;9fe0640122ec:32831 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-11T20:14:36,573 INFO  [RS:0;9fe0640122ec:32831 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-11T20:14:36,573 INFO  [RS:0;9fe0640122ec:32831 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-11T20:14:36,573 INFO  [RS:0;9fe0640122ec:32831 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-11T20:14:36,574 DEBUG [RS:0;9fe0640122ec:32831 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:14:36,574 DEBUG [RS:0;9fe0640122ec:32831 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:14:36,574 DEBUG [RS:0;9fe0640122ec:32831 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:14:36,574 DEBUG [RS:0;9fe0640122ec:32831 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:14:36,574 DEBUG [RS:0;9fe0640122ec:32831 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:14:36,574 DEBUG [RS:0;9fe0640122ec:32831 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9fe0640122ec:0, corePoolSize=2, maxPoolSize=2
2024-12-11T20:14:36,574 DEBUG [RS:0;9fe0640122ec:32831 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:14:36,574 DEBUG [RS:0;9fe0640122ec:32831 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:14:36,574 DEBUG [RS:0;9fe0640122ec:32831 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:14:36,574 DEBUG [RS:0;9fe0640122ec:32831 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:14:36,574 DEBUG [RS:0;9fe0640122ec:32831 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:14:36,574 DEBUG [RS:0;9fe0640122ec:32831 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:14:36,574 DEBUG [RS:0;9fe0640122ec:32831 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9fe0640122ec:0, corePoolSize=3, maxPoolSize=3
2024-12-11T20:14:36,574 DEBUG [RS:0;9fe0640122ec:32831 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9fe0640122ec:0, corePoolSize=3, maxPoolSize=3
2024-12-11T20:14:36,574 INFO  [RS:0;9fe0640122ec:32831 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-11T20:14:36,574 INFO  [RS:0;9fe0640122ec:32831 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-11T20:14:36,574 INFO  [RS:0;9fe0640122ec:32831 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-11T20:14:36,574 INFO  [RS:0;9fe0640122ec:32831 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-11T20:14:36,574 INFO  [RS:0;9fe0640122ec:32831 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-11T20:14:36,574 INFO  [RS:0;9fe0640122ec:32831 {}] hbase.ChoreService(168): Chore ScheduledChore name=9fe0640122ec,32831,1733948076323-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-11T20:14:36,590 INFO  [RS:0;9fe0640122ec:32831 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-11T20:14:36,590 INFO  [RS:0;9fe0640122ec:32831 {}] hbase.ChoreService(168): Chore ScheduledChore name=9fe0640122ec,32831,1733948076323-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-11T20:14:36,590 INFO  [RS:0;9fe0640122ec:32831 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T20:14:36,590 INFO  [RS:0;9fe0640122ec:32831 {}] regionserver.Replication(171): 9fe0640122ec,32831,1733948076323 started
2024-12-11T20:14:36,604 INFO  [RS:0;9fe0640122ec:32831 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T20:14:36,604 INFO  [RS:0;9fe0640122ec:32831 {}] regionserver.HRegionServer(1482): Serving as 9fe0640122ec,32831,1733948076323, RpcServer on 9fe0640122ec/172.17.0.2:32831, sessionid=0x100cafa036a0001
2024-12-11T20:14:36,604 DEBUG [RS:0;9fe0640122ec:32831 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-11T20:14:36,604 DEBUG [RS:0;9fe0640122ec:32831 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9fe0640122ec,32831,1733948076323
2024-12-11T20:14:36,604 DEBUG [RS:0;9fe0640122ec:32831 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9fe0640122ec,32831,1733948076323'
2024-12-11T20:14:36,604 DEBUG [RS:0;9fe0640122ec:32831 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-11T20:14:36,605 DEBUG [RS:0;9fe0640122ec:32831 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-11T20:14:36,605 DEBUG [RS:0;9fe0640122ec:32831 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-11T20:14:36,605 DEBUG [RS:0;9fe0640122ec:32831 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-11T20:14:36,605 DEBUG [RS:0;9fe0640122ec:32831 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9fe0640122ec,32831,1733948076323
2024-12-11T20:14:36,605 DEBUG [RS:0;9fe0640122ec:32831 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9fe0640122ec,32831,1733948076323'
2024-12-11T20:14:36,605 DEBUG [RS:0;9fe0640122ec:32831 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-11T20:14:36,606 DEBUG [RS:0;9fe0640122ec:32831 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-11T20:14:36,606 DEBUG [RS:0;9fe0640122ec:32831 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-11T20:14:36,606 INFO  [RS:0;9fe0640122ec:32831 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-11T20:14:36,606 INFO  [RS:0;9fe0640122ec:32831 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-11T20:14:36,648 WARN  [9fe0640122ec:46569 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-12-11T20:14:36,708 INFO  [RS:0;9fe0640122ec:32831 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9fe0640122ec%2C32831%2C1733948076323, suffix=, logDir=hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/WALs/9fe0640122ec,32831,1733948076323, archiveDir=hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/oldWALs, maxLogs=32
2024-12-11T20:14:36,709 INFO  [RS:0;9fe0640122ec:32831 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9fe0640122ec%2C32831%2C1733948076323.1733948076708
2024-12-11T20:14:36,714 INFO  [RS:0;9fe0640122ec:32831 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/WALs/9fe0640122ec,32831,1733948076323/9fe0640122ec%2C32831%2C1733948076323.1733948076708
2024-12-11T20:14:36,715 DEBUG [RS:0;9fe0640122ec:32831 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33307:33307),(127.0.0.1/127.0.0.1:39125:39125)]
2024-12-11T20:14:36,898 DEBUG [9fe0640122ec:46569 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-11T20:14:36,899 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=9fe0640122ec,32831,1733948076323
2024-12-11T20:14:36,900 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9fe0640122ec,32831,1733948076323, state=OPENING
2024-12-11T20:14:36,902 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-11T20:14:36,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46569-0x100cafa036a0000, quorum=127.0.0.1:53734, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:14:36,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32831-0x100cafa036a0001, quorum=127.0.0.1:53734, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:14:36,904 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-11T20:14:36,904 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-11T20:14:36,904 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=9fe0640122ec,32831,1733948076323}]
2024-12-11T20:14:36,904 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-11T20:14:37,057 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-11T20:14:37,059 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38199, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-11T20:14:37,063 INFO  [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-12-11T20:14:37,063 INFO  [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-11T20:14:37,065 INFO  [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9fe0640122ec%2C32831%2C1733948076323.meta, suffix=.meta, logDir=hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/WALs/9fe0640122ec,32831,1733948076323, archiveDir=hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/oldWALs, maxLogs=32
2024-12-11T20:14:37,065 INFO  [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 9fe0640122ec%2C32831%2C1733948076323.meta.1733948077065.meta
2024-12-11T20:14:37,071 INFO  [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/WALs/9fe0640122ec,32831,1733948076323/9fe0640122ec%2C32831%2C1733948076323.meta.1733948077065.meta
2024-12-11T20:14:37,074 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33307:33307),(127.0.0.1/127.0.0.1:39125:39125)]
2024-12-11T20:14:37,075 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-11T20:14:37,075 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-11T20:14:37,075 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-11T20:14:37,075 INFO  [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-11T20:14:37,075 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-11T20:14:37,075 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T20:14:37,075 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-12-11T20:14:37,075 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-12-11T20:14:37,076 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-11T20:14:37,077 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-11T20:14:37,077 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:14:37,078 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T20:14:37,078 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-11T20:14:37,078 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-11T20:14:37,078 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:14:37,079 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T20:14:37,079 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-11T20:14:37,079 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-11T20:14:37,079 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:14:37,080 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T20:14:37,080 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-11T20:14:37,080 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-11T20:14:37,080 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:14:37,081 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T20:14:37,081 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-11T20:14:37,081 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/hbase/meta/1588230740
2024-12-11T20:14:37,082 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/hbase/meta/1588230740
2024-12-11T20:14:37,083 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-11T20:14:37,083 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-11T20:14:37,084 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-11T20:14:37,085 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-11T20:14:37,085 INFO  [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=709084, jitterRate=-0.09835374355316162}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-11T20:14:37,086 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-12-11T20:14:37,086 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1733948077075Writing region info on filesystem at 1733948077075Initializing all the Stores at 1733948077076 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733948077076Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733948077076Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733948077076Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733948077076Cleaning up temporary data from old regions at 1733948077083 (+7 ms)Running coprocessor post-open hooks at 1733948077086 (+3 ms)Region opened successfully at 1733948077086
2024-12-11T20:14:37,087 INFO  [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733948077057
2024-12-11T20:14:37,089 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-11T20:14:37,089 INFO  [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-12-11T20:14:37,090 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=9fe0640122ec,32831,1733948076323
2024-12-11T20:14:37,091 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9fe0640122ec,32831,1733948076323, state=OPEN
2024-12-11T20:14:37,095 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46569-0x100cafa036a0000, quorum=127.0.0.1:53734, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-11T20:14:37,095 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32831-0x100cafa036a0001, quorum=127.0.0.1:53734, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-11T20:14:37,095 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=9fe0640122ec,32831,1733948076323
2024-12-11T20:14:37,095 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-11T20:14:37,095 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-11T20:14:37,098 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-12-11T20:14:37,098 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9fe0640122ec,32831,1733948076323 in 191 msec
2024-12-11T20:14:37,101 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-12-11T20:14:37,101 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 603 msec
2024-12-11T20:14:37,101 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-11T20:14:37,101 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-12-11T20:14:37,102 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-11T20:14:37,102 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9fe0640122ec,32831,1733948076323, seqNum=-1]
2024-12-11T20:14:37,103 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-11T20:14:37,104 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-15-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52643, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-11T20:14:37,108 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 646 msec
2024-12-11T20:14:37,108 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733948077108, completionTime=-1
2024-12-11T20:14:37,109 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-11T20:14:37,109 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-12-11T20:14:37,110 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-12-11T20:14:37,110 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733948137110
2024-12-11T20:14:37,110 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733948197110
2024-12-11T20:14:37,110 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 1 msec
2024-12-11T20:14:37,111 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9fe0640122ec,46569,1733948076281-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-11T20:14:37,111 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9fe0640122ec,46569,1733948076281-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T20:14:37,111 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9fe0640122ec,46569,1733948076281-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T20:14:37,111 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-9fe0640122ec:46569, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T20:14:37,111 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-11T20:14:37,111 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-12-11T20:14:37,113 DEBUG [master/9fe0640122ec:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-12-11T20:14:37,115 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.757sec
2024-12-11T20:14:37,115 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-11T20:14:37,115 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-11T20:14:37,115 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-11T20:14:37,115 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-11T20:14:37,115 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-11T20:14:37,115 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9fe0640122ec,46569,1733948076281-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-11T20:14:37,115 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9fe0640122ec,46569,1733948076281-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-11T20:14:37,117 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-12-11T20:14:37,117 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-11T20:14:37,117 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9fe0640122ec,46569,1733948076281-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T20:14:37,142 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57390027, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-11T20:14:37,142 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9fe0640122ec,46569,-1 for getting cluster id
2024-12-11T20:14:37,142 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-12-11T20:14:37,144 DEBUG [HMaster-EventLoopGroup-14-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b56e9c7a-9ff4-4371-9f7e-048538138346'
2024-12-11T20:14:37,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-12-11T20:14:37,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b56e9c7a-9ff4-4371-9f7e-048538138346"

2024-12-11T20:14:37,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@734465d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-11T20:14:37,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9fe0640122ec,46569,-1]
2024-12-11T20:14:37,145 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-12-11T20:14:37,145 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T20:14:37,146 INFO  [HMaster-EventLoopGroup-14-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38804, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-12-11T20:14:37,146 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@665a776, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-11T20:14:37,147 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-11T20:14:37,148 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9fe0640122ec,32831,1733948076323, seqNum=-1]
2024-12-11T20:14:37,148 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-11T20:14:37,149 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57776, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-11T20:14:37,150 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=9fe0640122ec,46569,1733948076281
2024-12-11T20:14:37,151 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T20:14:37,153 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-12-11T20:14:37,153 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-12-11T20:14:37,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 9fe0640122ec,46569,1733948076281
2024-12-11T20:14:37,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4c455bd6
2024-12-11T20:14:37,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-12-11T20:14:37,155 INFO  [HMaster-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38806, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-12-11T20:14:37,156 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46569 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-12-11T20:14:37,156 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46569 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-12-11T20:14:37,156 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46569 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-11T20:14:37,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46569 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRolling
2024-12-11T20:14:37,159 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-12-11T20:14:37,159 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:14:37,159 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46569 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRolling"
 procId is: 4
2024-12-11T20:14:37,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46569 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-11T20:14:37,160 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-11T20:14:37,170 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741835_1011 (size=381)
2024-12-11T20:14:37,170 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35725 is added to blk_1073741835_1011 (size=381)
2024-12-11T20:14:37,173 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 07f5c44b5ba0e29fbf0ec02f39733158, NAME => 'TestLogRolling-testLogRolling,,1733948077155.07f5c44b5ba0e29fbf0ec02f39733158.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c
2024-12-11T20:14:37,180 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35725 is added to blk_1073741836_1012 (size=64)
2024-12-11T20:14:37,181 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741836_1012 (size=64)
2024-12-11T20:14:37,181 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1733948077155.07f5c44b5ba0e29fbf0ec02f39733158.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T20:14:37,181 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1722): Closing 07f5c44b5ba0e29fbf0ec02f39733158, disabling compactions & flushes
2024-12-11T20:14:37,181 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1733948077155.07f5c44b5ba0e29fbf0ec02f39733158.
2024-12-11T20:14:37,181 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1733948077155.07f5c44b5ba0e29fbf0ec02f39733158.
2024-12-11T20:14:37,181 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1733948077155.07f5c44b5ba0e29fbf0ec02f39733158. after waiting 0 ms
2024-12-11T20:14:37,181 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1733948077155.07f5c44b5ba0e29fbf0ec02f39733158.
2024-12-11T20:14:37,181 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1733948077155.07f5c44b5ba0e29fbf0ec02f39733158.
2024-12-11T20:14:37,181 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for 07f5c44b5ba0e29fbf0ec02f39733158:
Waiting for close lock at 1733948077181Disabling compacts and flushes for region at 1733948077181Disabling writes for close at 1733948077181Writing region close event to WAL at 1733948077181Closed at 1733948077181
2024-12-11T20:14:37,183 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ADD_TO_META
2024-12-11T20:14:37,183 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRolling,,1733948077155.07f5c44b5ba0e29fbf0ec02f39733158.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1733948077183"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733948077183"}]},"ts":"1733948077183"}
2024-12-11T20:14:37,186 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-12-11T20:14:37,187 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-11T20:14:37,187 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733948077187"}]},"ts":"1733948077187"}
2024-12-11T20:14:37,189 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRolling, state=ENABLING in hbase:meta
2024-12-11T20:14:37,189 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=07f5c44b5ba0e29fbf0ec02f39733158, ASSIGN}]
2024-12-11T20:14:37,191 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=07f5c44b5ba0e29fbf0ec02f39733158, ASSIGN
2024-12-11T20:14:37,192 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=07f5c44b5ba0e29fbf0ec02f39733158, ASSIGN; state=OFFLINE, location=9fe0640122ec,32831,1733948076323; forceNewPlan=false, retain=false
2024-12-11T20:14:37,322 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:37,322 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:37,343 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=07f5c44b5ba0e29fbf0ec02f39733158, regionState=OPENING, regionLocation=9fe0640122ec,32831,1733948076323
2024-12-11T20:14:37,345 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=07f5c44b5ba0e29fbf0ec02f39733158, ASSIGN because future has completed
2024-12-11T20:14:37,346 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 07f5c44b5ba0e29fbf0ec02f39733158, server=9fe0640122ec,32831,1733948076323}]
2024-12-11T20:14:37,502 INFO  [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,,1733948077155.07f5c44b5ba0e29fbf0ec02f39733158.
2024-12-11T20:14:37,503 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 07f5c44b5ba0e29fbf0ec02f39733158, NAME => 'TestLogRolling-testLogRolling,,1733948077155.07f5c44b5ba0e29fbf0ec02f39733158.', STARTKEY => '', ENDKEY => ''}
2024-12-11T20:14:37,503 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling 07f5c44b5ba0e29fbf0ec02f39733158
2024-12-11T20:14:37,503 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1733948077155.07f5c44b5ba0e29fbf0ec02f39733158.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T20:14:37,503 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 07f5c44b5ba0e29fbf0ec02f39733158
2024-12-11T20:14:37,503 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 07f5c44b5ba0e29fbf0ec02f39733158
2024-12-11T20:14:37,504 INFO  [StoreOpener-07f5c44b5ba0e29fbf0ec02f39733158-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 07f5c44b5ba0e29fbf0ec02f39733158 
2024-12-11T20:14:37,506 INFO  [StoreOpener-07f5c44b5ba0e29fbf0ec02f39733158-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 07f5c44b5ba0e29fbf0ec02f39733158 columnFamilyName info
2024-12-11T20:14:37,506 DEBUG [StoreOpener-07f5c44b5ba0e29fbf0ec02f39733158-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:14:37,506 INFO  [StoreOpener-07f5c44b5ba0e29fbf0ec02f39733158-1 {}] regionserver.HStore(327): Store=07f5c44b5ba0e29fbf0ec02f39733158/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-11T20:14:37,506 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 07f5c44b5ba0e29fbf0ec02f39733158
2024-12-11T20:14:37,507 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158
2024-12-11T20:14:37,507 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158
2024-12-11T20:14:37,508 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 07f5c44b5ba0e29fbf0ec02f39733158
2024-12-11T20:14:37,508 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 07f5c44b5ba0e29fbf0ec02f39733158
2024-12-11T20:14:37,509 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 07f5c44b5ba0e29fbf0ec02f39733158
2024-12-11T20:14:37,511 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-11T20:14:37,511 INFO  [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 07f5c44b5ba0e29fbf0ec02f39733158; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=719922, jitterRate=-0.08457227051258087}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-11T20:14:37,511 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 07f5c44b5ba0e29fbf0ec02f39733158
2024-12-11T20:14:37,512 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 07f5c44b5ba0e29fbf0ec02f39733158:
Running coprocessor pre-open hook at 1733948077503Writing region info on filesystem at 1733948077503Initializing all the Stores at 1733948077504 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733948077504Cleaning up temporary data from old regions at 1733948077508 (+4 ms)Running coprocessor post-open hooks at 1733948077511 (+3 ms)Region opened successfully at 1733948077512 (+1 ms)
2024-12-11T20:14:37,513 INFO  [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,,1733948077155.07f5c44b5ba0e29fbf0ec02f39733158., pid=6, masterSystemTime=1733948077499
2024-12-11T20:14:37,515 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,,1733948077155.07f5c44b5ba0e29fbf0ec02f39733158.
2024-12-11T20:14:37,515 INFO  [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,,1733948077155.07f5c44b5ba0e29fbf0ec02f39733158.
2024-12-11T20:14:37,516 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=07f5c44b5ba0e29fbf0ec02f39733158, regionState=OPEN, openSeqNum=2, regionLocation=9fe0640122ec,32831,1733948076323
2024-12-11T20:14:37,518 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 07f5c44b5ba0e29fbf0ec02f39733158, server=9fe0640122ec,32831,1733948076323 because future has completed
2024-12-11T20:14:37,522 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-12-11T20:14:37,522 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 07f5c44b5ba0e29fbf0ec02f39733158, server=9fe0640122ec,32831,1733948076323 in 174 msec
2024-12-11T20:14:37,525 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-12-11T20:14:37,525 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=07f5c44b5ba0e29fbf0ec02f39733158, ASSIGN in 333 msec
2024-12-11T20:14:37,526 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-11T20:14:37,526 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733948077526"}]},"ts":"1733948077526"}
2024-12-11T20:14:37,528 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRolling, state=ENABLED in hbase:meta
2024-12-11T20:14:37,529 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_POST_OPERATION
2024-12-11T20:14:37,531 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRolling in 373 msec
2024-12-11T20:14:38,323 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:38,323 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:39,324 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:39,324 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:39,747 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:14:39,747 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:14:39,747 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:14:39,747 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:14:39,747 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:14:39,747 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:14:39,748 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:14:39,748 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:14:39,760 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:14:39,761 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:14:39,761 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:14:39,761 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:14:39,761 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:14:39,761 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:14:39,764 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:14:39,764 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:14:39,764 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:14:39,766 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:14:40,270 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-11T20:14:40,271 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:14:40,271 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:14:40,271 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:14:40,271 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:14:40,272 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:14:40,272 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:14:40,272 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:14:40,272 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:14:40,289 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:14:40,289 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:14:40,289 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:14:40,289 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:14:40,289 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:14:40,290 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:14:40,293 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:14:40,293 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:14:40,294 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:14:40,296 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:14:40,324 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:40,324 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:41,325 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:41,325 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:42,156 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling
2024-12-11T20:14:42,156 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling Metrics about Tables on a single HBase RegionServer
2024-12-11T20:14:42,156 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-11T20:14:42,325 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:42,325 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:42,570 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-12-11T20:14:42,570 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRolling'
2024-12-11T20:14:43,326 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:43,326 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:44,327 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:44,327 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:45,327 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:45,327 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:46,328 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:46,328 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:47,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46569 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-11T20:14:47,178 INFO  [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRolling completed
2024-12-11T20:14:47,178 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRolling,, stopping at row=TestLogRolling-testLogRolling ,, for max=2147483647 with caching=100
2024-12-11T20:14:47,180 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRolling
2024-12-11T20:14:47,181 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRolling,,1733948077155.07f5c44b5ba0e29fbf0ec02f39733158.
2024-12-11T20:14:47,183 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testLogRolling,,1733948077155.07f5c44b5ba0e29fbf0ec02f39733158., hostname=9fe0640122ec,32831,1733948076323, seqNum=2]
2024-12-11T20:14:47,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32831 {}] regionserver.HRegion(8855): Flush requested on 07f5c44b5ba0e29fbf0ec02f39733158
2024-12-11T20:14:47,194 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 07f5c44b5ba0e29fbf0ec02f39733158 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-11T20:14:47,211 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/.tmp/info/da43a63ffdf849c1b80f7c262f7b9d62 is 1080, key is row0001/info:/1733948087184/Put/seqid=0
2024-12-11T20:14:47,217 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35725 is added to blk_1073741837_1013 (size=12509)
2024-12-11T20:14:47,217 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741837_1013 (size=12509)
2024-12-11T20:14:47,218 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/.tmp/info/da43a63ffdf849c1b80f7c262f7b9d62
2024-12-11T20:14:47,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/.tmp/info/da43a63ffdf849c1b80f7c262f7b9d62 as hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/info/da43a63ffdf849c1b80f7c262f7b9d62
2024-12-11T20:14:47,232 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32831 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit.
org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=07f5c44b5ba0e29fbf0ec02f39733158, server=9fe0640122ec,32831,1733948076323
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT]
2024-12-11T20:14:47,232 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/info/da43a63ffdf849c1b80f7c262f7b9d62, entries=7, sequenceid=11, filesize=12.2 K
2024-12-11T20:14:47,233 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=23.12 KB/23672 for 07f5c44b5ba0e29fbf0ec02f39733158 in 39ms, sequenceid=11, compaction requested=false
2024-12-11T20:14:47,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 07f5c44b5ba0e29fbf0ec02f39733158:

2024-12-11T20:14:47,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32831 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:57776 deadline: 1733948097231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=07f5c44b5ba0e29fbf0ec02f39733158, server=9fe0640122ec,32831,1733948076323
2024-12-11T20:14:47,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestLogRolling-testLogRolling,,1733948077155.07f5c44b5ba0e29fbf0ec02f39733158., hostname=9fe0640122ec,32831,1733948076323, seqNum=2 , the old value is region=TestLogRolling-testLogRolling,,1733948077155.07f5c44b5ba0e29fbf0ec02f39733158., hostname=9fe0640122ec,32831,1733948076323, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=07f5c44b5ba0e29fbf0ec02f39733158, server=9fe0640122ec,32831,1733948076323
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308)
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329)
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216)
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-12-11T20:14:47,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestLogRolling-testLogRolling,,1733948077155.07f5c44b5ba0e29fbf0ec02f39733158., hostname=9fe0640122ec,32831,1733948076323, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=07f5c44b5ba0e29fbf0ec02f39733158, server=9fe0640122ec,32831,1733948076323
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308)
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329)
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216)
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-12-11T20:14:47,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestLogRolling-testLogRolling,,1733948077155.07f5c44b5ba0e29fbf0ec02f39733158., hostname=9fe0640122ec,32831,1733948076323, seqNum=2 because the exception is null or not the one we care about
2024-12-11T20:14:47,328 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:47,328 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:47,659 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-11T20:14:47,660 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:14:47,660 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:14:47,661 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:14:47,661 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:14:47,661 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:14:47,661 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:14:47,662 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:14:47,663 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:14:47,680 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:14:47,680 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:14:47,680 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:14:47,681 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:14:47,681 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:14:47,681 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:14:47,684 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:14:47,684 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:14:47,685 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:14:47,687 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:14:48,329 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:48,329 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:49,330 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:49,330 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:50,330 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:50,330 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:51,331 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:51,331 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:52,332 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:52,332 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:53,332 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:53,332 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:54,333 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:54,333 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:55,334 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:55,333 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:56,334 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:56,334 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:57,335 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:57,335 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:57,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32831 {}] regionserver.HRegion(8855): Flush requested on 07f5c44b5ba0e29fbf0ec02f39733158
2024-12-11T20:14:57,349 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 07f5c44b5ba0e29fbf0ec02f39733158 1/1 column families, dataSize=24.17 KB heapSize=26.13 KB
2024-12-11T20:14:57,354 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/.tmp/info/51f13726cb4848efb5c8b08b967914ef is 1080, key is row0008/info:/1733948087195/Put/seqid=0
2024-12-11T20:14:57,359 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741838_1014 (size=29761)
2024-12-11T20:14:57,359 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35725 is added to blk_1073741838_1014 (size=29761)
2024-12-11T20:14:57,359 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.17 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/.tmp/info/51f13726cb4848efb5c8b08b967914ef
2024-12-11T20:14:57,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/.tmp/info/51f13726cb4848efb5c8b08b967914ef as hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/info/51f13726cb4848efb5c8b08b967914ef
2024-12-11T20:14:57,369 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/info/51f13726cb4848efb5c8b08b967914ef, entries=23, sequenceid=37, filesize=29.1 K
2024-12-11T20:14:57,371 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~24.17 KB/24748, heapSize ~26.11 KB/26736, currentSize=2.10 KB/2152 for 07f5c44b5ba0e29fbf0ec02f39733158 in 22ms, sequenceid=37, compaction requested=false
2024-12-11T20:14:57,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 07f5c44b5ba0e29fbf0ec02f39733158:

2024-12-11T20:14:57,371 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=41.3 K, sizeToCheck=16.0 K
2024-12-11T20:14:57,371 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-11T20:14:57,371 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/info/51f13726cb4848efb5c8b08b967914ef because midkey is the same as first or last row
2024-12-11T20:14:58,335 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:58,335 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:59,336 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:59,336 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:14:59,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32831 {}] regionserver.HRegion(8855): Flush requested on 07f5c44b5ba0e29fbf0ec02f39733158
2024-12-11T20:14:59,362 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 07f5c44b5ba0e29fbf0ec02f39733158 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-11T20:14:59,367 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/.tmp/info/3f85f105fdf74815969fe63fd547b787 is 1080, key is row0031/info:/1733948097350/Put/seqid=0
2024-12-11T20:14:59,372 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35725 is added to blk_1073741839_1015 (size=12509)
2024-12-11T20:14:59,372 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741839_1015 (size=12509)
2024-12-11T20:14:59,373 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/.tmp/info/3f85f105fdf74815969fe63fd547b787
2024-12-11T20:14:59,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/.tmp/info/3f85f105fdf74815969fe63fd547b787 as hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/info/3f85f105fdf74815969fe63fd547b787
2024-12-11T20:14:59,385 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/info/3f85f105fdf74815969fe63fd547b787, entries=7, sequenceid=47, filesize=12.2 K
2024-12-11T20:14:59,386 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=13.66 KB/13988 for 07f5c44b5ba0e29fbf0ec02f39733158 in 23ms, sequenceid=47, compaction requested=true
2024-12-11T20:14:59,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 07f5c44b5ba0e29fbf0ec02f39733158:

2024-12-11T20:14:59,386 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=53.5 K, sizeToCheck=16.0 K
2024-12-11T20:14:59,386 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-11T20:14:59,386 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/info/51f13726cb4848efb5c8b08b967914ef because midkey is the same as first or last row
2024-12-11T20:14:59,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07f5c44b5ba0e29fbf0ec02f39733158:info, priority=-2147483648, current under compaction store size is 1
2024-12-11T20:14:59,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-11T20:14:59,386 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-11T20:14:59,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32831 {}] regionserver.HRegion(8855): Flush requested on 07f5c44b5ba0e29fbf0ec02f39733158
2024-12-11T20:14:59,387 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 07f5c44b5ba0e29fbf0ec02f39733158 1/1 column families, dataSize=14.71 KB heapSize=16 KB
2024-12-11T20:14:59,388 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 54779 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-11T20:14:59,388 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.HStore(1541): 07f5c44b5ba0e29fbf0ec02f39733158/info is initiating minor compaction (all files)
2024-12-11T20:14:59,388 INFO  [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07f5c44b5ba0e29fbf0ec02f39733158/info in TestLogRolling-testLogRolling,,1733948077155.07f5c44b5ba0e29fbf0ec02f39733158.
2024-12-11T20:14:59,388 INFO  [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/info/da43a63ffdf849c1b80f7c262f7b9d62, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/info/51f13726cb4848efb5c8b08b967914ef, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/info/3f85f105fdf74815969fe63fd547b787] into tmpdir=hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/.tmp, totalSize=53.5 K
2024-12-11T20:14:59,389 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] compactions.Compactor(225): Compacting da43a63ffdf849c1b80f7c262f7b9d62, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733948087184
2024-12-11T20:14:59,389 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51f13726cb4848efb5c8b08b967914ef, keycount=23, bloomtype=ROW, size=29.1 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1733948087195
2024-12-11T20:14:59,389 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f85f105fdf74815969fe63fd547b787, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733948097350
2024-12-11T20:14:59,392 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/.tmp/info/f2df0d4b8795480ab823bc0554cb4469 is 1080, key is row0038/info:/1733948099363/Put/seqid=0
2024-12-11T20:14:59,397 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35725 is added to blk_1073741840_1016 (size=20064)
2024-12-11T20:14:59,398 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741840_1016 (size=20064)
2024-12-11T20:14:59,398 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.71 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/.tmp/info/f2df0d4b8795480ab823bc0554cb4469
2024-12-11T20:14:59,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/.tmp/info/f2df0d4b8795480ab823bc0554cb4469 as hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/info/f2df0d4b8795480ab823bc0554cb4469
2024-12-11T20:14:59,410 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/info/f2df0d4b8795480ab823bc0554cb4469, entries=14, sequenceid=64, filesize=19.6 K
2024-12-11T20:14:59,411 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~14.71 KB/15064, heapSize ~15.98 KB/16368, currentSize=12.61 KB/12912 for 07f5c44b5ba0e29fbf0ec02f39733158 in 25ms, sequenceid=64, compaction requested=false
2024-12-11T20:14:59,411 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 07f5c44b5ba0e29fbf0ec02f39733158:

2024-12-11T20:14:59,411 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=73.1 K, sizeToCheck=16.0 K
2024-12-11T20:14:59,411 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-11T20:14:59,411 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/info/51f13726cb4848efb5c8b08b967914ef because midkey is the same as first or last row
2024-12-11T20:14:59,412 INFO  [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07f5c44b5ba0e29fbf0ec02f39733158#info#compaction#60 average throughput is 18.98 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-11T20:14:59,412 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/.tmp/info/8a81c2f736224c7aa7f6c406c552fbda is 1080, key is row0001/info:/1733948087184/Put/seqid=0
2024-12-11T20:14:59,417 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741841_1017 (size=44978)
2024-12-11T20:14:59,417 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35725 is added to blk_1073741841_1017 (size=44978)
2024-12-11T20:14:59,423 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/.tmp/info/8a81c2f736224c7aa7f6c406c552fbda as hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/info/8a81c2f736224c7aa7f6c406c552fbda
2024-12-11T20:14:59,429 INFO  [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 07f5c44b5ba0e29fbf0ec02f39733158/info of 07f5c44b5ba0e29fbf0ec02f39733158 into 8a81c2f736224c7aa7f6c406c552fbda(size=43.9 K), total size for store is 63.5 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-11T20:14:59,429 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07f5c44b5ba0e29fbf0ec02f39733158:

2024-12-11T20:14:59,429 INFO  [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1733948077155.07f5c44b5ba0e29fbf0ec02f39733158., storeName=07f5c44b5ba0e29fbf0ec02f39733158/info, priority=13, startTime=1733948099386; duration=0sec
2024-12-11T20:14:59,429 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=63.5 K, sizeToCheck=16.0 K
2024-12-11T20:14:59,429 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-11T20:14:59,429 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/info/8a81c2f736224c7aa7f6c406c552fbda because midkey is the same as first or last row
2024-12-11T20:14:59,429 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=63.5 K, sizeToCheck=16.0 K
2024-12-11T20:14:59,429 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-11T20:14:59,430 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/info/8a81c2f736224c7aa7f6c406c552fbda because midkey is the same as first or last row
2024-12-11T20:14:59,430 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=63.5 K, sizeToCheck=16.0 K
2024-12-11T20:14:59,430 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-11T20:14:59,430 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/info/8a81c2f736224c7aa7f6c406c552fbda because midkey is the same as first or last row
2024-12-11T20:14:59,430 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-11T20:14:59,430 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07f5c44b5ba0e29fbf0ec02f39733158:info
2024-12-11T20:15:00,337 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:15:00,337 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:15:01,337 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:15:01,337 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:15:01,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32831 {}] regionserver.HRegion(8855): Flush requested on 07f5c44b5ba0e29fbf0ec02f39733158
2024-12-11T20:15:01,414 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 07f5c44b5ba0e29fbf0ec02f39733158 1/1 column families, dataSize=14.71 KB heapSize=16 KB
2024-12-11T20:15:01,418 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/.tmp/info/f42f25a8a1714a79829ef9cf54722afd is 1080, key is row0052/info:/1733948099388/Put/seqid=0
2024-12-11T20:15:01,423 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741842_1018 (size=20064)
2024-12-11T20:15:01,424 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35725 is added to blk_1073741842_1018 (size=20064)
2024-12-11T20:15:01,425 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.71 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/.tmp/info/f42f25a8a1714a79829ef9cf54722afd
2024-12-11T20:15:01,431 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/.tmp/info/f42f25a8a1714a79829ef9cf54722afd as hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/info/f42f25a8a1714a79829ef9cf54722afd
2024-12-11T20:15:01,437 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/info/f42f25a8a1714a79829ef9cf54722afd, entries=14, sequenceid=82, filesize=19.6 K
2024-12-11T20:15:01,439 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~14.71 KB/15064, heapSize ~15.98 KB/16368, currentSize=14.71 KB/15064 for 07f5c44b5ba0e29fbf0ec02f39733158 in 24ms, sequenceid=82, compaction requested=true
2024-12-11T20:15:01,439 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 07f5c44b5ba0e29fbf0ec02f39733158:

2024-12-11T20:15:01,439 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=83.1 K, sizeToCheck=16.0 K
2024-12-11T20:15:01,439 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-11T20:15:01,439 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/info/8a81c2f736224c7aa7f6c406c552fbda because midkey is the same as first or last row
2024-12-11T20:15:01,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 07f5c44b5ba0e29fbf0ec02f39733158:info, priority=-2147483648, current under compaction store size is 1
2024-12-11T20:15:01,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-11T20:15:01,439 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-11T20:15:01,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32831 {}] regionserver.HRegion(8855): Flush requested on 07f5c44b5ba0e29fbf0ec02f39733158
2024-12-11T20:15:01,440 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 07f5c44b5ba0e29fbf0ec02f39733158 1/1 column families, dataSize=15.76 KB heapSize=17.13 KB
2024-12-11T20:15:01,441 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 85106 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-11T20:15:01,441 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.HStore(1541): 07f5c44b5ba0e29fbf0ec02f39733158/info is initiating minor compaction (all files)
2024-12-11T20:15:01,441 INFO  [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 07f5c44b5ba0e29fbf0ec02f39733158/info in TestLogRolling-testLogRolling,,1733948077155.07f5c44b5ba0e29fbf0ec02f39733158.
2024-12-11T20:15:01,441 INFO  [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/info/8a81c2f736224c7aa7f6c406c552fbda, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/info/f2df0d4b8795480ab823bc0554cb4469, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/info/f42f25a8a1714a79829ef9cf54722afd] into tmpdir=hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/.tmp, totalSize=83.1 K
2024-12-11T20:15:01,441 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a81c2f736224c7aa7f6c406c552fbda, keycount=37, bloomtype=ROW, size=43.9 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733948087184
2024-12-11T20:15:01,442 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2df0d4b8795480ab823bc0554cb4469, keycount=14, bloomtype=ROW, size=19.6 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733948099363
2024-12-11T20:15:01,442 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] compactions.Compactor(225): Compacting f42f25a8a1714a79829ef9cf54722afd, keycount=14, bloomtype=ROW, size=19.6 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733948099388
2024-12-11T20:15:01,444 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/.tmp/info/686f9b3bdb70430b955f58147222b7c9 is 1080, key is row0066/info:/1733948101415/Put/seqid=0
2024-12-11T20:15:01,449 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35725 is added to blk_1073741843_1019 (size=21141)
2024-12-11T20:15:01,450 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741843_1019 (size=21141)
2024-12-11T20:15:01,450 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.76 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/.tmp/info/686f9b3bdb70430b955f58147222b7c9
2024-12-11T20:15:01,456 INFO  [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 07f5c44b5ba0e29fbf0ec02f39733158#info#compaction#63 average throughput is 22.23 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-11T20:15:01,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/.tmp/info/686f9b3bdb70430b955f58147222b7c9 as hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/info/686f9b3bdb70430b955f58147222b7c9
2024-12-11T20:15:01,457 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/.tmp/info/588b5cca3d0c4abba65d01b91f92cc1b is 1080, key is row0001/info:/1733948087184/Put/seqid=0
2024-12-11T20:15:01,461 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741844_1020 (size=75378)
2024-12-11T20:15:01,462 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35725 is added to blk_1073741844_1020 (size=75378)
2024-12-11T20:15:01,463 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/info/686f9b3bdb70430b955f58147222b7c9, entries=15, sequenceid=100, filesize=20.6 K
2024-12-11T20:15:01,464 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~15.76 KB/16140, heapSize ~17.11 KB/17520, currentSize=12.61 KB/12912 for 07f5c44b5ba0e29fbf0ec02f39733158 in 24ms, sequenceid=100, compaction requested=false
2024-12-11T20:15:01,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 07f5c44b5ba0e29fbf0ec02f39733158:

2024-12-11T20:15:01,464 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=103.8 K, sizeToCheck=16.0 K
2024-12-11T20:15:01,464 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-11T20:15:01,464 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/info/8a81c2f736224c7aa7f6c406c552fbda because midkey is the same as first or last row
2024-12-11T20:15:01,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32831 {}] regionserver.HRegion(8855): Flush requested on 07f5c44b5ba0e29fbf0ec02f39733158
2024-12-11T20:15:01,466 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 07f5c44b5ba0e29fbf0ec02f39733158 1/1 column families, dataSize=13.66 KB heapSize=14.88 KB
2024-12-11T20:15:01,469 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/.tmp/info/588b5cca3d0c4abba65d01b91f92cc1b as hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/info/588b5cca3d0c4abba65d01b91f92cc1b
2024-12-11T20:15:01,471 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/.tmp/info/cf43db9cd99c41e3bdad620556077e4b is 1080, key is row0081/info:/1733948101441/Put/seqid=0
2024-12-11T20:15:01,475 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741845_1021 (size=18987)
2024-12-11T20:15:01,476 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35725 is added to blk_1073741845_1021 (size=18987)
2024-12-11T20:15:01,476 INFO  [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 07f5c44b5ba0e29fbf0ec02f39733158/info of 07f5c44b5ba0e29fbf0ec02f39733158 into 588b5cca3d0c4abba65d01b91f92cc1b(size=73.6 K), total size for store is 94.3 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-11T20:15:01,476 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.66 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/.tmp/info/cf43db9cd99c41e3bdad620556077e4b
2024-12-11T20:15:01,476 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 07f5c44b5ba0e29fbf0ec02f39733158:

2024-12-11T20:15:01,476 INFO  [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1733948077155.07f5c44b5ba0e29fbf0ec02f39733158., storeName=07f5c44b5ba0e29fbf0ec02f39733158/info, priority=13, startTime=1733948101439; duration=0sec
2024-12-11T20:15:01,476 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=94.3 K, sizeToCheck=16.0 K
2024-12-11T20:15:01,476 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-11T20:15:01,476 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=94.3 K, sizeToCheck=16.0 K
2024-12-11T20:15:01,476 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-11T20:15:01,477 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=94.3 K, sizeToCheck=16.0 K
2024-12-11T20:15:01,477 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-11T20:15:01,478 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.CompactSplit(239): Splitting TestLogRolling-testLogRolling,,1733948077155.07f5c44b5ba0e29fbf0ec02f39733158., compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-11T20:15:01,478 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-11T20:15:01,478 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 07f5c44b5ba0e29fbf0ec02f39733158:info
2024-12-11T20:15:01,479 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46569 {}] assignment.AssignmentManager(1363): Split request from 9fe0640122ec,32831,1733948076323, parent={ENCODED => 07f5c44b5ba0e29fbf0ec02f39733158, NAME => 'TestLogRolling-testLogRolling,,1733948077155.07f5c44b5ba0e29fbf0ec02f39733158.', STARTKEY => '', ENDKEY => ''}, splitKey=row0062
2024-12-11T20:15:01,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/.tmp/info/cf43db9cd99c41e3bdad620556077e4b as hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/info/cf43db9cd99c41e3bdad620556077e4b
2024-12-11T20:15:01,484 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46569 {}] assignment.SplitTableRegionProcedure(223): Splittable=true state=OPEN, location=9fe0640122ec,32831,1733948076323
2024-12-11T20:15:01,486 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/info/cf43db9cd99c41e3bdad620556077e4b, entries=13, sequenceid=116, filesize=18.5 K
2024-12-11T20:15:01,488 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.66 KB/13988, heapSize ~14.86 KB/15216, currentSize=3.15 KB/3228 for 07f5c44b5ba0e29fbf0ec02f39733158 in 22ms, sequenceid=116, compaction requested=true
2024-12-11T20:15:01,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 07f5c44b5ba0e29fbf0ec02f39733158:

2024-12-11T20:15:01,488 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=112.8 K, sizeToCheck=16.0 K
2024-12-11T20:15:01,488 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-11T20:15:01,488 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=112.8 K, sizeToCheck=16.0 K
2024-12-11T20:15:01,488 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-11T20:15:01,488 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=112.8 K, sizeToCheck=16.0 K
2024-12-11T20:15:01,488 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-11T20:15:01,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(239): Splitting TestLogRolling-testLogRolling,,1733948077155.07f5c44b5ba0e29fbf0ec02f39733158., compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=1
2024-12-11T20:15:01,488 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46569 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=07f5c44b5ba0e29fbf0ec02f39733158, daughterA=607a8af8bf8fceec868c0a5c1d5fefa1, daughterB=0000cbe68f7bb76d4869b7889b4f2fec
2024-12-11T20:15:01,490 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=07f5c44b5ba0e29fbf0ec02f39733158, daughterA=607a8af8bf8fceec868c0a5c1d5fefa1, daughterB=0000cbe68f7bb76d4869b7889b4f2fec
2024-12-11T20:15:01,490 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=07f5c44b5ba0e29fbf0ec02f39733158, daughterA=607a8af8bf8fceec868c0a5c1d5fefa1, daughterB=0000cbe68f7bb76d4869b7889b4f2fec
2024-12-11T20:15:01,490 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=07f5c44b5ba0e29fbf0ec02f39733158, daughterA=607a8af8bf8fceec868c0a5c1d5fefa1, daughterB=0000cbe68f7bb76d4869b7889b4f2fec
2024-12-11T20:15:01,491 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46569 {}] assignment.AssignmentManager(1363): Split request from 9fe0640122ec,32831,1733948076323, parent={ENCODED => 07f5c44b5ba0e29fbf0ec02f39733158, NAME => 'TestLogRolling-testLogRolling,,1733948077155.07f5c44b5ba0e29fbf0ec02f39733158.', STARTKEY => '', ENDKEY => ''}, splitKey=row0062
2024-12-11T20:15:01,492 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46569 {}] assignment.SplitTableRegionProcedure(223): Splittable=true state=SPLITTING, location=9fe0640122ec,32831,1733948076323
2024-12-11T20:15:01,493 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46569 {}] procedure2.ProcedureExecutor(1139): Stored pid=8, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=07f5c44b5ba0e29fbf0ec02f39733158, daughterA=8cd400880b1ddab09907de41f2f1ae4b, daughterB=2775d423e72ed67b0bba4d57d71e3c8a
2024-12-11T20:15:01,493 INFO  [PEWorker-3 {}] procedure.MasterProcedureScheduler(842): Waiting on xlock for pid=8, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=07f5c44b5ba0e29fbf0ec02f39733158, daughterA=8cd400880b1ddab09907de41f2f1ae4b, daughterB=2775d423e72ed67b0bba4d57d71e3c8a held by pid=7
2024-12-11T20:15:01,498 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=07f5c44b5ba0e29fbf0ec02f39733158, UNASSIGN}]
2024-12-11T20:15:01,501 DEBUG [PEWorker-3 {}] assignment.SplitTableRegionProcedure(162): LOCK_EVENT_WAIT SchemaLocking[serverLocks={},namespaceLocks={hbase=LockAndQueue[exclusiveLock=false,sharedLockCount=0,waitingProcCount=0]},tableLocks={hbase:meta=LockAndQueue[exclusiveLock=false,sharedLockCount=0,waitingProcCount=0]},regionLocks={},peerLocks={},metaLocks={hbase:meta=LockAndQueue[exclusiveLock=false,sharedLockCount=0,waitingProcCount=0]},globalLocks={}]
2024-12-11T20:15:01,501 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=07f5c44b5ba0e29fbf0ec02f39733158, UNASSIGN
2024-12-11T20:15:01,501 DEBUG [PEWorker-3 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=8, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=07f5c44b5ba0e29fbf0ec02f39733158, daughterA=8cd400880b1ddab09907de41f2f1ae4b, daughterB=2775d423e72ed67b0bba4d57d71e3c8a
2024-12-11T20:15:01,502 INFO  [PEWorker-4 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=07f5c44b5ba0e29fbf0ec02f39733158, regionState=CLOSING, regionLocation=9fe0640122ec,32831,1733948076323
2024-12-11T20:15:01,504 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=07f5c44b5ba0e29fbf0ec02f39733158, UNASSIGN because future has completed
2024-12-11T20:15:01,505 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: true: evictOnSplit: true: evictOnClose: false
2024-12-11T20:15:01,505 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; CloseRegionProcedure 07f5c44b5ba0e29fbf0ec02f39733158, server=9fe0640122ec,32831,1733948076323}]
2024-12-11T20:15:01,662 INFO  [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] handler.UnassignRegionHandler(122): Close 07f5c44b5ba0e29fbf0ec02f39733158
2024-12-11T20:15:01,662 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] handler.UnassignRegionHandler(136): Unassign region: split region: true: evictCache: true
2024-12-11T20:15:01,663 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1722): Closing 07f5c44b5ba0e29fbf0ec02f39733158, disabling compactions & flushes
2024-12-11T20:15:01,663 INFO  [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1733948077155.07f5c44b5ba0e29fbf0ec02f39733158.
2024-12-11T20:15:01,663 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1733948077155.07f5c44b5ba0e29fbf0ec02f39733158.
2024-12-11T20:15:01,663 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1733948077155.07f5c44b5ba0e29fbf0ec02f39733158. after waiting 0 ms
2024-12-11T20:15:01,663 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1733948077155.07f5c44b5ba0e29fbf0ec02f39733158.
2024-12-11T20:15:01,663 INFO  [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(2902): Flushing 07f5c44b5ba0e29fbf0ec02f39733158 1/1 column families, dataSize=3.15 KB heapSize=3.63 KB
2024-12-11T20:15:01,668 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/.tmp/info/f1b31f7a36a54d389928d5566883f385 is 1080, key is row0094/info:/1733948101467/Put/seqid=0
2024-12-11T20:15:01,672 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35725 is added to blk_1073741846_1022 (size=8193)
2024-12-11T20:15:01,672 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741846_1022 (size=8193)
2024-12-11T20:15:01,673 INFO  [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.15 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/.tmp/info/f1b31f7a36a54d389928d5566883f385
2024-12-11T20:15:01,678 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/.tmp/info/f1b31f7a36a54d389928d5566883f385 as hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/info/f1b31f7a36a54d389928d5566883f385
2024-12-11T20:15:01,683 INFO  [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/info/f1b31f7a36a54d389928d5566883f385, entries=3, sequenceid=123, filesize=8.0 K
2024-12-11T20:15:01,684 INFO  [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 07f5c44b5ba0e29fbf0ec02f39733158 in 21ms, sequenceid=123, compaction requested=true
2024-12-11T20:15:01,685 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733948077155.07f5c44b5ba0e29fbf0ec02f39733158.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/info/da43a63ffdf849c1b80f7c262f7b9d62, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/info/51f13726cb4848efb5c8b08b967914ef, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/info/8a81c2f736224c7aa7f6c406c552fbda, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/info/3f85f105fdf74815969fe63fd547b787, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/info/f2df0d4b8795480ab823bc0554cb4469, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/info/f42f25a8a1714a79829ef9cf54722afd] to archive
2024-12-11T20:15:01,686 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733948077155.07f5c44b5ba0e29fbf0ec02f39733158.-1 {}] backup.HFileArchiver(363): Archiving compacted files.
2024-12-11T20:15:01,688 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/info/da43a63ffdf849c1b80f7c262f7b9d62 to hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/archive/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/info/da43a63ffdf849c1b80f7c262f7b9d62
2024-12-11T20:15:01,688 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/info/8a81c2f736224c7aa7f6c406c552fbda to hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/archive/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/info/8a81c2f736224c7aa7f6c406c552fbda
2024-12-11T20:15:01,688 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/info/51f13726cb4848efb5c8b08b967914ef to hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/archive/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/info/51f13726cb4848efb5c8b08b967914ef
2024-12-11T20:15:01,689 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/info/3f85f105fdf74815969fe63fd547b787 to hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/archive/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/info/3f85f105fdf74815969fe63fd547b787
2024-12-11T20:15:01,689 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/info/f42f25a8a1714a79829ef9cf54722afd to hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/archive/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/info/f42f25a8a1714a79829ef9cf54722afd
2024-12-11T20:15:01,689 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/info/f2df0d4b8795480ab823bc0554cb4469 to hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/archive/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/info/f2df0d4b8795480ab823bc0554cb4469
2024-12-11T20:15:01,695 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/recovered.edits/126.seqid, newMaxSeqId=126, maxSeqId=1
2024-12-11T20:15:01,696 INFO  [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1733948077155.07f5c44b5ba0e29fbf0ec02f39733158.
2024-12-11T20:15:01,697 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1676): Region close journal for 07f5c44b5ba0e29fbf0ec02f39733158:
Waiting for close lock at 1733948101663Running coprocessor pre-close hooks at 1733948101663Disabling compacts and flushes for region at 1733948101663Disabling writes for close at 1733948101663Obtaining lock to block concurrent updates at 1733948101663Preparing flush snapshotting stores in 07f5c44b5ba0e29fbf0ec02f39733158 at 1733948101663Finished memstore snapshotting TestLogRolling-testLogRolling,,1733948077155.07f5c44b5ba0e29fbf0ec02f39733158., syncing WAL and waiting on mvcc, flushsize=dataSize=3228, getHeapSize=3696, getOffHeapSize=0, getCellsCount=3 at 1733948101664 (+1 ms)Flushing stores of TestLogRolling-testLogRolling,,1733948077155.07f5c44b5ba0e29fbf0ec02f39733158. at 1733948101664Flushing 07f5c44b5ba0e29fbf0ec02f39733158/info: creating writer at 1733948101664Flushing 07f5c44b5ba0e29fbf0ec02f39733158/info: appending metadata at 1733948101667 (+3 ms)Flushing 07f5c44b5ba0e29fbf0ec02f39733158/info: closing flushed file at 1733948101667Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@138b2897: reopening flushed file at 1733948101677 (+10 ms)Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 07f5c44b5ba0e29fbf0ec02f39733158 in 21ms, sequenceid=123, compaction requested=true at 1733948101684 (+7 ms)Writing region close event to WAL at 1733948101692 (+8 ms)Running coprocessor post-close hooks at 1733948101696 (+4 ms)Closed at 1733948101696
2024-12-11T20:15:01,699 INFO  [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] handler.UnassignRegionHandler(157): Closed 07f5c44b5ba0e29fbf0ec02f39733158
2024-12-11T20:15:01,699 INFO  [PEWorker-2 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=07f5c44b5ba0e29fbf0ec02f39733158, regionState=CLOSED
2024-12-11T20:15:01,701 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=9, state=RUNNABLE, hasLock=false; CloseRegionProcedure 07f5c44b5ba0e29fbf0ec02f39733158, server=9fe0640122ec,32831,1733948076323 because future has completed
2024-12-11T20:15:01,705 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9
2024-12-11T20:15:01,705 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; CloseRegionProcedure 07f5c44b5ba0e29fbf0ec02f39733158, server=9fe0640122ec,32831,1733948076323 in 198 msec
2024-12-11T20:15:01,708 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7
2024-12-11T20:15:01,708 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=07f5c44b5ba0e29fbf0ec02f39733158, UNASSIGN in 207 msec
2024-12-11T20:15:01,716 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:15:01,723 INFO  [PEWorker-5 {}] assignment.SplitTableRegionProcedure(728): pid=7 splitting 4 storefiles, region=07f5c44b5ba0e29fbf0ec02f39733158, threads=4
2024-12-11T20:15:01,725 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/info/686f9b3bdb70430b955f58147222b7c9 for region: 07f5c44b5ba0e29fbf0ec02f39733158
2024-12-11T20:15:01,725 DEBUG [StoreFileSplitter-pool-3 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/info/f1b31f7a36a54d389928d5566883f385 for region: 07f5c44b5ba0e29fbf0ec02f39733158
2024-12-11T20:15:01,725 DEBUG [StoreFileSplitter-pool-2 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/info/cf43db9cd99c41e3bdad620556077e4b for region: 07f5c44b5ba0e29fbf0ec02f39733158
2024-12-11T20:15:01,725 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/info/588b5cca3d0c4abba65d01b91f92cc1b for region: 07f5c44b5ba0e29fbf0ec02f39733158
2024-12-11T20:15:01,735 DEBUG [StoreFileSplitter-pool-2 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/info/cf43db9cd99c41e3bdad620556077e4b, top=true
2024-12-11T20:15:01,737 DEBUG [StoreFileSplitter-pool-1 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/info/686f9b3bdb70430b955f58147222b7c9, top=true
2024-12-11T20:15:01,738 DEBUG [StoreFileSplitter-pool-3 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/info/f1b31f7a36a54d389928d5566883f385, top=true
2024-12-11T20:15:01,744 INFO  [StoreFileSplitter-pool-2 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/TestLogRolling-testLogRolling=07f5c44b5ba0e29fbf0ec02f39733158-cf43db9cd99c41e3bdad620556077e4b for child: 0000cbe68f7bb76d4869b7889b4f2fec, parent: 07f5c44b5ba0e29fbf0ec02f39733158
2024-12-11T20:15:01,744 DEBUG [StoreFileSplitter-pool-2 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/info/cf43db9cd99c41e3bdad620556077e4b for region: 07f5c44b5ba0e29fbf0ec02f39733158
2024-12-11T20:15:01,745 INFO  [StoreFileSplitter-pool-3 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/TestLogRolling-testLogRolling=07f5c44b5ba0e29fbf0ec02f39733158-f1b31f7a36a54d389928d5566883f385 for child: 0000cbe68f7bb76d4869b7889b4f2fec, parent: 07f5c44b5ba0e29fbf0ec02f39733158
2024-12-11T20:15:01,745 DEBUG [StoreFileSplitter-pool-3 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/info/f1b31f7a36a54d389928d5566883f385 for region: 07f5c44b5ba0e29fbf0ec02f39733158
2024-12-11T20:15:01,745 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35725 is added to blk_1073741847_1023 (size=27)
2024-12-11T20:15:01,745 INFO  [StoreFileSplitter-pool-1 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/TestLogRolling-testLogRolling=07f5c44b5ba0e29fbf0ec02f39733158-686f9b3bdb70430b955f58147222b7c9 for child: 0000cbe68f7bb76d4869b7889b4f2fec, parent: 07f5c44b5ba0e29fbf0ec02f39733158
2024-12-11T20:15:01,745 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/info/686f9b3bdb70430b955f58147222b7c9 for region: 07f5c44b5ba0e29fbf0ec02f39733158
2024-12-11T20:15:01,746 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741847_1023 (size=27)
2024-12-11T20:15:01,756 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741848_1024 (size=27)
2024-12-11T20:15:01,756 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35725 is added to blk_1073741848_1024 (size=27)
2024-12-11T20:15:01,757 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/info/588b5cca3d0c4abba65d01b91f92cc1b for region: 07f5c44b5ba0e29fbf0ec02f39733158
2024-12-11T20:15:01,759 DEBUG [PEWorker-5 {}] assignment.SplitTableRegionProcedure(802): pid=7 split storefiles for region 07f5c44b5ba0e29fbf0ec02f39733158 Daughter A: [hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/607a8af8bf8fceec868c0a5c1d5fefa1/info/588b5cca3d0c4abba65d01b91f92cc1b.07f5c44b5ba0e29fbf0ec02f39733158] storefiles, Daughter B: [hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/588b5cca3d0c4abba65d01b91f92cc1b.07f5c44b5ba0e29fbf0ec02f39733158, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/TestLogRolling-testLogRolling=07f5c44b5ba0e29fbf0ec02f39733158-686f9b3bdb70430b955f58147222b7c9, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/TestLogRolling-testLogRolling=07f5c44b5ba0e29fbf0ec02f39733158-cf43db9cd99c41e3bdad620556077e4b, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/TestLogRolling-testLogRolling=07f5c44b5ba0e29fbf0ec02f39733158-f1b31f7a36a54d389928d5566883f385] storefiles.
2024-12-11T20:15:01,766 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741849_1025 (size=71)
2024-12-11T20:15:01,767 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35725 is added to blk_1073741849_1025 (size=71)
2024-12-11T20:15:01,769 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:15:01,781 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35725 is added to blk_1073741850_1026 (size=71)
2024-12-11T20:15:01,781 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741850_1026 (size=71)
2024-12-11T20:15:01,783 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:15:01,794 DEBUG [PEWorker-5 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/607a8af8bf8fceec868c0a5c1d5fefa1/recovered.edits/126.seqid, newMaxSeqId=126, maxSeqId=-1
2024-12-11T20:15:01,796 DEBUG [PEWorker-5 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/recovered.edits/126.seqid, newMaxSeqId=126, maxSeqId=-1
2024-12-11T20:15:01,798 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1733948077155.07f5c44b5ba0e29fbf0ec02f39733158.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1733948101798"},{"qualifier":"splitA","vlen":70,"tag":[],"timestamp":"1733948101798"},{"qualifier":"splitB","vlen":70,"tag":[],"timestamp":"1733948101798"}]},"ts":"1733948101798"}
2024-12-11T20:15:01,799 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1733948101484.607a8af8bf8fceec868c0a5c1d5fefa1.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1733948101798"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733948101798"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1733948101798"}]},"ts":"1733948101798"}
2024-12-11T20:15:01,799 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,row0062,1733948101484.0000cbe68f7bb76d4869b7889b4f2fec.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1733948101798"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733948101798"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1733948101798"}]},"ts":"1733948101798"}
2024-12-11T20:15:01,818 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=607a8af8bf8fceec868c0a5c1d5fefa1, ASSIGN}, {pid=12, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=0000cbe68f7bb76d4869b7889b4f2fec, ASSIGN}]
2024-12-11T20:15:01,819 INFO  [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=607a8af8bf8fceec868c0a5c1d5fefa1, ASSIGN
2024-12-11T20:15:01,819 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=0000cbe68f7bb76d4869b7889b4f2fec, ASSIGN
2024-12-11T20:15:01,820 INFO  [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=607a8af8bf8fceec868c0a5c1d5fefa1, ASSIGN; state=SPLITTING_NEW, location=9fe0640122ec,32831,1733948076323; forceNewPlan=false, retain=false
2024-12-11T20:15:01,820 INFO  [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=0000cbe68f7bb76d4869b7889b4f2fec, ASSIGN; state=SPLITTING_NEW, location=9fe0640122ec,32831,1733948076323; forceNewPlan=false, retain=false
2024-12-11T20:15:01,971 INFO  [PEWorker-4 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=607a8af8bf8fceec868c0a5c1d5fefa1, regionState=OPENING, regionLocation=9fe0640122ec,32831,1733948076323
2024-12-11T20:15:01,971 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=0000cbe68f7bb76d4869b7889b4f2fec, regionState=OPENING, regionLocation=9fe0640122ec,32831,1733948076323
2024-12-11T20:15:01,973 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=607a8af8bf8fceec868c0a5c1d5fefa1, ASSIGN because future has completed
2024-12-11T20:15:01,974 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=13, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 607a8af8bf8fceec868c0a5c1d5fefa1, server=9fe0640122ec,32831,1733948076323}]
2024-12-11T20:15:01,974 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=0000cbe68f7bb76d4869b7889b4f2fec, ASSIGN because future has completed
2024-12-11T20:15:01,975 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0000cbe68f7bb76d4869b7889b4f2fec, server=9fe0640122ec,32831,1733948076323}]
2024-12-11T20:15:02,130 INFO  [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,row0062,1733948101484.0000cbe68f7bb76d4869b7889b4f2fec.
2024-12-11T20:15:02,130 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(7752): Opening region: {ENCODED => 0000cbe68f7bb76d4869b7889b4f2fec, NAME => 'TestLogRolling-testLogRolling,row0062,1733948101484.0000cbe68f7bb76d4869b7889b4f2fec.', STARTKEY => 'row0062', ENDKEY => ''}
2024-12-11T20:15:02,131 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling 0000cbe68f7bb76d4869b7889b4f2fec
2024-12-11T20:15:02,131 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,row0062,1733948101484.0000cbe68f7bb76d4869b7889b4f2fec.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T20:15:02,131 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(7794): checking encryption for 0000cbe68f7bb76d4869b7889b4f2fec
2024-12-11T20:15:02,131 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(7797): checking classloading for 0000cbe68f7bb76d4869b7889b4f2fec
2024-12-11T20:15:02,132 INFO  [StoreOpener-0000cbe68f7bb76d4869b7889b4f2fec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 0000cbe68f7bb76d4869b7889b4f2fec 
2024-12-11T20:15:02,133 INFO  [StoreOpener-0000cbe68f7bb76d4869b7889b4f2fec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0000cbe68f7bb76d4869b7889b4f2fec columnFamilyName info
2024-12-11T20:15:02,133 DEBUG [StoreOpener-0000cbe68f7bb76d4869b7889b4f2fec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:15:02,143 DEBUG [StoreOpener-0000cbe68f7bb76d4869b7889b4f2fec-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/588b5cca3d0c4abba65d01b91f92cc1b.07f5c44b5ba0e29fbf0ec02f39733158->hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/info/588b5cca3d0c4abba65d01b91f92cc1b-top
2024-12-11T20:15:02,148 DEBUG [StoreOpener-0000cbe68f7bb76d4869b7889b4f2fec-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/TestLogRolling-testLogRolling=07f5c44b5ba0e29fbf0ec02f39733158-686f9b3bdb70430b955f58147222b7c9
2024-12-11T20:15:02,152 DEBUG [StoreOpener-0000cbe68f7bb76d4869b7889b4f2fec-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/TestLogRolling-testLogRolling=07f5c44b5ba0e29fbf0ec02f39733158-cf43db9cd99c41e3bdad620556077e4b
2024-12-11T20:15:02,155 DEBUG [StoreOpener-0000cbe68f7bb76d4869b7889b4f2fec-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/TestLogRolling-testLogRolling=07f5c44b5ba0e29fbf0ec02f39733158-f1b31f7a36a54d389928d5566883f385
2024-12-11T20:15:02,156 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling
2024-12-11T20:15:02,156 INFO  [StoreOpener-0000cbe68f7bb76d4869b7889b4f2fec-1 {}] regionserver.HStore(327): Store=0000cbe68f7bb76d4869b7889b4f2fec/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-11T20:15:02,156 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(1038): replaying wal for 0000cbe68f7bb76d4869b7889b4f2fec
2024-12-11T20:15:02,157 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec
2024-12-11T20:15:02,157 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec
2024-12-11T20:15:02,158 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(1048): stopping wal replay for 0000cbe68f7bb76d4869b7889b4f2fec
2024-12-11T20:15:02,158 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(1060): Cleaning up temporary data for 0000cbe68f7bb76d4869b7889b4f2fec
2024-12-11T20:15:02,160 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(1093): writing seq id for 0000cbe68f7bb76d4869b7889b4f2fec
2024-12-11T20:15:02,160 INFO  [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(1114): Opened 0000cbe68f7bb76d4869b7889b4f2fec; next sequenceid=127; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=700191, jitterRate=-0.10966186225414276}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-11T20:15:02,160 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0000cbe68f7bb76d4869b7889b4f2fec
2024-12-11T20:15:02,161 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(1006): Region open journal for 0000cbe68f7bb76d4869b7889b4f2fec:
Running coprocessor pre-open hook at 1733948102131Writing region info on filesystem at 1733948102131Initializing all the Stores at 1733948102131Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733948102132 (+1 ms)Cleaning up temporary data from old regions at 1733948102158 (+26 ms)Running coprocessor post-open hooks at 1733948102160 (+2 ms)Region opened successfully at 1733948102161 (+1 ms)
2024-12-11T20:15:02,162 INFO  [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,row0062,1733948101484.0000cbe68f7bb76d4869b7889b4f2fec., pid=14, masterSystemTime=1733948102126
2024-12-11T20:15:02,162 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.CompactSplit(403): Add compact mark for store 0000cbe68f7bb76d4869b7889b4f2fec:info, priority=-2147483648, current under compaction store size is 1
2024-12-11T20:15:02,162 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking
2024-12-11T20:15:02,162 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-11T20:15:02,164 INFO  [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.HStore(1527): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,row0062,1733948101484.0000cbe68f7bb76d4869b7889b4f2fec.
2024-12-11T20:15:02,164 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.HStore(1541): 0000cbe68f7bb76d4869b7889b4f2fec/info is initiating minor compaction (all files)
2024-12-11T20:15:02,164 INFO  [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0000cbe68f7bb76d4869b7889b4f2fec/info in TestLogRolling-testLogRolling,row0062,1733948101484.0000cbe68f7bb76d4869b7889b4f2fec.
2024-12-11T20:15:02,164 INFO  [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/588b5cca3d0c4abba65d01b91f92cc1b.07f5c44b5ba0e29fbf0ec02f39733158->hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/info/588b5cca3d0c4abba65d01b91f92cc1b-top, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/TestLogRolling-testLogRolling=07f5c44b5ba0e29fbf0ec02f39733158-686f9b3bdb70430b955f58147222b7c9, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/TestLogRolling-testLogRolling=07f5c44b5ba0e29fbf0ec02f39733158-cf43db9cd99c41e3bdad620556077e4b, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/TestLogRolling-testLogRolling=07f5c44b5ba0e29fbf0ec02f39733158-f1b31f7a36a54d389928d5566883f385] into tmpdir=hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/.tmp, totalSize=120.8 K
2024-12-11T20:15:02,164 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,row0062,1733948101484.0000cbe68f7bb76d4869b7889b4f2fec.
2024-12-11T20:15:02,164 INFO  [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,row0062,1733948101484.0000cbe68f7bb76d4869b7889b4f2fec.
2024-12-11T20:15:02,164 INFO  [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,,1733948101484.607a8af8bf8fceec868c0a5c1d5fefa1.
2024-12-11T20:15:02,165 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7752): Opening region: {ENCODED => 607a8af8bf8fceec868c0a5c1d5fefa1, NAME => 'TestLogRolling-testLogRolling,,1733948101484.607a8af8bf8fceec868c0a5c1d5fefa1.', STARTKEY => '', ENDKEY => 'row0062'}
2024-12-11T20:15:02,165 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 588b5cca3d0c4abba65d01b91f92cc1b.07f5c44b5ba0e29fbf0ec02f39733158, keycount=32, bloomtype=ROW, size=73.6 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733948087184
2024-12-11T20:15:02,165 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling 607a8af8bf8fceec868c0a5c1d5fefa1
2024-12-11T20:15:02,165 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1733948101484.607a8af8bf8fceec868c0a5c1d5fefa1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T20:15:02,165 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7794): checking encryption for 607a8af8bf8fceec868c0a5c1d5fefa1
2024-12-11T20:15:02,165 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7797): checking classloading for 607a8af8bf8fceec868c0a5c1d5fefa1
2024-12-11T20:15:02,165 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] compactions.Compactor(225): Compacting TestLogRolling-testLogRolling=07f5c44b5ba0e29fbf0ec02f39733158-686f9b3bdb70430b955f58147222b7c9, keycount=15, bloomtype=ROW, size=20.6 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1733948101415
2024-12-11T20:15:02,165 INFO  [PEWorker-4 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=0000cbe68f7bb76d4869b7889b4f2fec, regionState=OPEN, openSeqNum=127, regionLocation=9fe0640122ec,32831,1733948076323
2024-12-11T20:15:02,165 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] compactions.Compactor(225): Compacting TestLogRolling-testLogRolling=07f5c44b5ba0e29fbf0ec02f39733158-cf43db9cd99c41e3bdad620556077e4b, keycount=13, bloomtype=ROW, size=18.5 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1733948101441
2024-12-11T20:15:02,166 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] compactions.Compactor(225): Compacting TestLogRolling-testLogRolling=07f5c44b5ba0e29fbf0ec02f39733158-f1b31f7a36a54d389928d5566883f385, keycount=3, bloomtype=ROW, size=8.0 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1733948101467
2024-12-11T20:15:02,166 INFO  [StoreOpener-607a8af8bf8fceec868c0a5c1d5fefa1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 607a8af8bf8fceec868c0a5c1d5fefa1 
2024-12-11T20:15:02,167 INFO  [StoreOpener-607a8af8bf8fceec868c0a5c1d5fefa1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 607a8af8bf8fceec868c0a5c1d5fefa1 columnFamilyName info
2024-12-11T20:15:02,167 DEBUG [StoreOpener-607a8af8bf8fceec868c0a5c1d5fefa1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:15:02,167 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32831 {}] regionserver.HRegion(8855): Flush requested on 1588230740
2024-12-11T20:15:02,167 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all.
2024-12-11T20:15:02,167 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.15 KB heapSize=9 KB
2024-12-11T20:15:02,167 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0000cbe68f7bb76d4869b7889b4f2fec, server=9fe0640122ec,32831,1733948076323 because future has completed
2024-12-11T20:15:02,172 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=12
2024-12-11T20:15:02,172 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure 0000cbe68f7bb76d4869b7889b4f2fec, server=9fe0640122ec,32831,1733948076323 in 194 msec
2024-12-11T20:15:02,174 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=0000cbe68f7bb76d4869b7889b4f2fec, ASSIGN in 354 msec
2024-12-11T20:15:02,177 DEBUG [StoreOpener-607a8af8bf8fceec868c0a5c1d5fefa1-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/607a8af8bf8fceec868c0a5c1d5fefa1/info/588b5cca3d0c4abba65d01b91f92cc1b.07f5c44b5ba0e29fbf0ec02f39733158->hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/info/588b5cca3d0c4abba65d01b91f92cc1b-bottom
2024-12-11T20:15:02,178 INFO  [StoreOpener-607a8af8bf8fceec868c0a5c1d5fefa1-1 {}] regionserver.HStore(327): Store=607a8af8bf8fceec868c0a5c1d5fefa1/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-11T20:15:02,178 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1038): replaying wal for 607a8af8bf8fceec868c0a5c1d5fefa1
2024-12-11T20:15:02,179 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/607a8af8bf8fceec868c0a5c1d5fefa1
2024-12-11T20:15:02,180 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/607a8af8bf8fceec868c0a5c1d5fefa1
2024-12-11T20:15:02,180 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1048): stopping wal replay for 607a8af8bf8fceec868c0a5c1d5fefa1
2024-12-11T20:15:02,180 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1060): Cleaning up temporary data for 607a8af8bf8fceec868c0a5c1d5fefa1
2024-12-11T20:15:02,182 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1093): writing seq id for 607a8af8bf8fceec868c0a5c1d5fefa1
2024-12-11T20:15:02,183 INFO  [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1114): Opened 607a8af8bf8fceec868c0a5c1d5fefa1; next sequenceid=127; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=763579, jitterRate=-0.029059767723083496}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-11T20:15:02,183 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 607a8af8bf8fceec868c0a5c1d5fefa1
2024-12-11T20:15:02,183 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1006): Region open journal for 607a8af8bf8fceec868c0a5c1d5fefa1:
Running coprocessor pre-open hook at 1733948102165Writing region info on filesystem at 1733948102165Initializing all the Stores at 1733948102166 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733948102166Cleaning up temporary data from old regions at 1733948102180 (+14 ms)Running coprocessor post-open hooks at 1733948102183 (+3 ms)Region opened successfully at 1733948102183
2024-12-11T20:15:02,184 INFO  [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,,1733948101484.607a8af8bf8fceec868c0a5c1d5fefa1., pid=13, masterSystemTime=1733948102126
2024-12-11T20:15:02,184 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.CompactSplit(403): Add compact mark for store 607a8af8bf8fceec868c0a5c1d5fefa1:info, priority=-2147483648, current under compaction store size is 2
2024-12-11T20:15:02,184 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-11T20:15:02,184 DEBUG [RS:0;9fe0640122ec:32831-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking
2024-12-11T20:15:02,185 INFO  [RS:0;9fe0640122ec:32831-longCompactions-0 {}] regionserver.HStore(1527): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,,1733948101484.607a8af8bf8fceec868c0a5c1d5fefa1.
2024-12-11T20:15:02,186 DEBUG [RS:0;9fe0640122ec:32831-longCompactions-0 {}] regionserver.HStore(1541): 607a8af8bf8fceec868c0a5c1d5fefa1/info is initiating minor compaction (all files)
2024-12-11T20:15:02,186 INFO  [RS:0;9fe0640122ec:32831-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 607a8af8bf8fceec868c0a5c1d5fefa1/info in TestLogRolling-testLogRolling,,1733948101484.607a8af8bf8fceec868c0a5c1d5fefa1.
2024-12-11T20:15:02,186 INFO  [RS:0;9fe0640122ec:32831-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/607a8af8bf8fceec868c0a5c1d5fefa1/info/588b5cca3d0c4abba65d01b91f92cc1b.07f5c44b5ba0e29fbf0ec02f39733158->hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/info/588b5cca3d0c4abba65d01b91f92cc1b-bottom] into tmpdir=hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/607a8af8bf8fceec868c0a5c1d5fefa1/.tmp, totalSize=73.6 K
2024-12-11T20:15:02,186 DEBUG [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,,1733948101484.607a8af8bf8fceec868c0a5c1d5fefa1.
2024-12-11T20:15:02,186 INFO  [RS_OPEN_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,,1733948101484.607a8af8bf8fceec868c0a5c1d5fefa1.
2024-12-11T20:15:02,187 DEBUG [RS:0;9fe0640122ec:32831-longCompactions-0 {}] compactions.Compactor(225): Compacting 588b5cca3d0c4abba65d01b91f92cc1b.07f5c44b5ba0e29fbf0ec02f39733158, keycount=32, bloomtype=ROW, size=73.6 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733948087184
2024-12-11T20:15:02,187 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=607a8af8bf8fceec868c0a5c1d5fefa1, regionState=OPEN, openSeqNum=127, regionLocation=9fe0640122ec,32831,1733948076323
2024-12-11T20:15:02,188 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/hbase/meta/1588230740/.tmp/info/988fa4cc819e40658db16ed71119991d is 193, key is TestLogRolling-testLogRolling,row0062,1733948101484.0000cbe68f7bb76d4869b7889b4f2fec./info:regioninfo/1733948102165/Put/seqid=0
2024-12-11T20:15:02,190 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 607a8af8bf8fceec868c0a5c1d5fefa1, server=9fe0640122ec,32831,1733948076323 because future has completed
2024-12-11T20:15:02,193 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741851_1027 (size=9882)
2024-12-11T20:15:02,194 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35725 is added to blk_1073741851_1027 (size=9882)
2024-12-11T20:15:02,194 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=13, resume processing ppid=11
2024-12-11T20:15:02,194 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 607a8af8bf8fceec868c0a5c1d5fefa1, server=9fe0640122ec,32831,1733948076323 in 217 msec
2024-12-11T20:15:02,197 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=7
2024-12-11T20:15:02,197 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.95 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/hbase/meta/1588230740/.tmp/info/988fa4cc819e40658db16ed71119991d
2024-12-11T20:15:02,197 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=607a8af8bf8fceec868c0a5c1d5fefa1, ASSIGN in 376 msec
2024-12-11T20:15:02,199 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=07f5c44b5ba0e29fbf0ec02f39733158, daughterA=607a8af8bf8fceec868c0a5c1d5fefa1, daughterB=0000cbe68f7bb76d4869b7889b4f2fec in 713 msec
2024-12-11T20:15:02,199 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=07f5c44b5ba0e29fbf0ec02f39733158, daughterA=8cd400880b1ddab09907de41f2f1ae4b, daughterB=2775d423e72ed67b0bba4d57d71e3c8a
2024-12-11T20:15:02,199 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=07f5c44b5ba0e29fbf0ec02f39733158, daughterA=8cd400880b1ddab09907de41f2f1ae4b, daughterB=2775d423e72ed67b0bba4d57d71e3c8a
2024-12-11T20:15:02,199 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=07f5c44b5ba0e29fbf0ec02f39733158, daughterA=8cd400880b1ddab09907de41f2f1ae4b, daughterB=2775d423e72ed67b0bba4d57d71e3c8a
2024-12-11T20:15:02,200 INFO  [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0000cbe68f7bb76d4869b7889b4f2fec#info#compaction#67 average throughput is 17.96 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-11T20:15:02,200 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/.tmp/info/ca10a55495df44b8ae8e3e09c5b6eac4 is 1080, key is row0062/info:/1733948099407/Put/seqid=0
2024-12-11T20:15:02,201 INFO  [PEWorker-1 {}] assignment.SplitTableRegionProcedure(534): Split of {ENCODED => 07f5c44b5ba0e29fbf0ec02f39733158, NAME => 'TestLogRolling-testLogRolling,,1733948077155.07f5c44b5ba0e29fbf0ec02f39733158.', STARTKEY => '', ENDKEY => ''} skipped; state is already SPLIT
2024-12-11T20:15:02,202 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, state=SUCCESS, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=07f5c44b5ba0e29fbf0ec02f39733158, daughterA=8cd400880b1ddab09907de41f2f1ae4b, daughterB=2775d423e72ed67b0bba4d57d71e3c8a in 709 msec
2024-12-11T20:15:02,210 INFO  [RS:0;9fe0640122ec:32831-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 607a8af8bf8fceec868c0a5c1d5fefa1#info#compaction#68 average throughput is 31.30 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-11T20:15:02,211 DEBUG [RS:0;9fe0640122ec:32831-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/607a8af8bf8fceec868c0a5c1d5fefa1/.tmp/info/dda88a9ea4384d14af5e1d5cdeccb057 is 1080, key is row0001/info:/1733948087184/Put/seqid=0
2024-12-11T20:15:02,214 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741852_1028 (size=43081)
2024-12-11T20:15:02,214 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35725 is added to blk_1073741852_1028 (size=43081)
2024-12-11T20:15:02,220 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/.tmp/info/ca10a55495df44b8ae8e3e09c5b6eac4 as hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/ca10a55495df44b8ae8e3e09c5b6eac4
2024-12-11T20:15:02,221 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741853_1029 (size=70862)
2024-12-11T20:15:02,222 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35725 is added to blk_1073741853_1029 (size=70862)
2024-12-11T20:15:02,225 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/hbase/meta/1588230740/.tmp/ns/f8c91d208d19461db4ab8c78a984e2bb is 43, key is default/ns:d/1733948077104/Put/seqid=0
2024-12-11T20:15:02,227 DEBUG [RS:0;9fe0640122ec:32831-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/607a8af8bf8fceec868c0a5c1d5fefa1/.tmp/info/dda88a9ea4384d14af5e1d5cdeccb057 as hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/607a8af8bf8fceec868c0a5c1d5fefa1/info/dda88a9ea4384d14af5e1d5cdeccb057
2024-12-11T20:15:02,228 INFO  [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0000cbe68f7bb76d4869b7889b4f2fec/info of 0000cbe68f7bb76d4869b7889b4f2fec into ca10a55495df44b8ae8e3e09c5b6eac4(size=42.1 K), total size for store is 42.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-11T20:15:02,228 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0000cbe68f7bb76d4869b7889b4f2fec:

2024-12-11T20:15:02,228 INFO  [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733948101484.0000cbe68f7bb76d4869b7889b4f2fec., storeName=0000cbe68f7bb76d4869b7889b4f2fec/info, priority=12, startTime=1733948102162; duration=0sec
2024-12-11T20:15:02,228 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-11T20:15:02,228 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0000cbe68f7bb76d4869b7889b4f2fec:info
2024-12-11T20:15:02,230 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741854_1030 (size=5153)
2024-12-11T20:15:02,231 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35725 is added to blk_1073741854_1030 (size=5153)
2024-12-11T20:15:02,231 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/hbase/meta/1588230740/.tmp/ns/f8c91d208d19461db4ab8c78a984e2bb
2024-12-11T20:15:02,234 INFO  [RS:0;9fe0640122ec:32831-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 1 (all) file(s) in 607a8af8bf8fceec868c0a5c1d5fefa1/info of 607a8af8bf8fceec868c0a5c1d5fefa1 into dda88a9ea4384d14af5e1d5cdeccb057(size=69.2 K), total size for store is 69.2 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-11T20:15:02,234 DEBUG [RS:0;9fe0640122ec:32831-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 607a8af8bf8fceec868c0a5c1d5fefa1:

2024-12-11T20:15:02,234 INFO  [RS:0;9fe0640122ec:32831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1733948101484.607a8af8bf8fceec868c0a5c1d5fefa1., storeName=607a8af8bf8fceec868c0a5c1d5fefa1/info, priority=15, startTime=1733948102184; duration=0sec
2024-12-11T20:15:02,234 DEBUG [RS:0;9fe0640122ec:32831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-11T20:15:02,234 DEBUG [RS:0;9fe0640122ec:32831-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 607a8af8bf8fceec868c0a5c1d5fefa1:info
2024-12-11T20:15:02,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/hbase/meta/1588230740/.tmp/table/6f7911bd35f2460ea65d6d27f6968d8f is 65, key is TestLogRolling-testLogRolling/table:state/1733948077526/Put/seqid=0
2024-12-11T20:15:02,254 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741855_1031 (size=5340)
2024-12-11T20:15:02,255 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35725 is added to blk_1073741855_1031 (size=5340)
2024-12-11T20:15:02,255 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=122 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/hbase/meta/1588230740/.tmp/table/6f7911bd35f2460ea65d6d27f6968d8f
2024-12-11T20:15:02,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/hbase/meta/1588230740/.tmp/info/988fa4cc819e40658db16ed71119991d as hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/hbase/meta/1588230740/info/988fa4cc819e40658db16ed71119991d
2024-12-11T20:15:02,265 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/hbase/meta/1588230740/info/988fa4cc819e40658db16ed71119991d, entries=30, sequenceid=17, filesize=9.7 K
2024-12-11T20:15:02,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/hbase/meta/1588230740/.tmp/ns/f8c91d208d19461db4ab8c78a984e2bb as hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/hbase/meta/1588230740/ns/f8c91d208d19461db4ab8c78a984e2bb
2024-12-11T20:15:02,271 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/hbase/meta/1588230740/ns/f8c91d208d19461db4ab8c78a984e2bb, entries=2, sequenceid=17, filesize=5.0 K
2024-12-11T20:15:02,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/hbase/meta/1588230740/.tmp/table/6f7911bd35f2460ea65d6d27f6968d8f as hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/hbase/meta/1588230740/table/6f7911bd35f2460ea65d6d27f6968d8f
2024-12-11T20:15:02,276 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/hbase/meta/1588230740/table/6f7911bd35f2460ea65d6d27f6968d8f, entries=2, sequenceid=17, filesize=5.2 K
2024-12-11T20:15:02,277 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~5.15 KB/5269, heapSize ~8.70 KB/8912, currentSize=670 B/670 for 1588230740 in 110ms, sequenceid=17, compaction requested=false
2024-12-11T20:15:02,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1588230740:

2024-12-11T20:15:02,338 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:15:02,338 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:15:03,338 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:15:03,338 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:15:03,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32831 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:57776 deadline: 1733948113473, exception=org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1733948077155.07f5c44b5ba0e29fbf0ec02f39733158. is not online on 9fe0640122ec,32831,1733948076323
2024-12-11T20:15:03,474 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestLogRolling-testLogRolling,,1733948077155.07f5c44b5ba0e29fbf0ec02f39733158., hostname=9fe0640122ec,32831,1733948076323, seqNum=2 , the old value is region=TestLogRolling-testLogRolling,,1733948077155.07f5c44b5ba0e29fbf0ec02f39733158., hostname=9fe0640122ec,32831,1733948076323, seqNum=2, error=org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1733948077155.07f5c44b5ba0e29fbf0ec02f39733158. is not online on 9fe0640122ec,32831,1733948076323
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186)
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2943)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-12-11T20:15:03,474 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestLogRolling-testLogRolling,,1733948077155.07f5c44b5ba0e29fbf0ec02f39733158., hostname=9fe0640122ec,32831,1733948076323, seqNum=2 is org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1733948077155.07f5c44b5ba0e29fbf0ec02f39733158. is not online on 9fe0640122ec,32831,1733948076323
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186)
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2943)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-12-11T20:15:03,475 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=TestLogRolling-testLogRolling,,1733948077155.07f5c44b5ba0e29fbf0ec02f39733158., hostname=9fe0640122ec,32831,1733948076323, seqNum=2 from cache
2024-12-11T20:15:04,339 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:15:04,339 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:15:05,340 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:15:05,340 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:15:06,265 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-11T20:15:06,340 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:15:06,340 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:15:06,697 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:15:06,697 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:15:06,697 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:15:06,697 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:15:06,697 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:15:06,698 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:15:06,698 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:15:06,698 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:15:06,714 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:15:06,714 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:15:06,714 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:15:06,715 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:15:06,715 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:15:06,715 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:15:06,718 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:15:06,718 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:15:06,718 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:15:06,720 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:15:07,227 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-11T20:15:07,228 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:15:07,228 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:15:07,228 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:15:07,228 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:15:07,229 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:15:07,229 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:15:07,229 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:15:07,230 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:15:07,247 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:15:07,247 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:15:07,247 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:15:07,248 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:15:07,248 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:15:07,248 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:15:07,251 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:15:07,251 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:15:07,252 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:15:07,254 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-11T20:15:07,341 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:15:07,341 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:15:08,341 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:15:08,341 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:15:09,342 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:15:09,342 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:15:10,343 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:15:10,343 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:15:11,343 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:15:11,343 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:15:12,156 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling
2024-12-11T20:15:12,156 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling Metrics about Tables on a single HBase RegionServer
2024-12-11T20:15:12,344 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:15:12,344 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:15:13,345 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:15:13,345 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:15:13,545 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRolling', row='row0097', locateType=CURRENT is [region=TestLogRolling-testLogRolling,row0062,1733948101484.0000cbe68f7bb76d4869b7889b4f2fec., hostname=9fe0640122ec,32831,1733948076323, seqNum=127]
2024-12-11T20:15:13,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32831 {}] regionserver.HRegion(8855): Flush requested on 0000cbe68f7bb76d4869b7889b4f2fec
2024-12-11T20:15:13,557 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0000cbe68f7bb76d4869b7889b4f2fec 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-11T20:15:13,561 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/.tmp/info/1255ca8191fa4d6a84e31726bb9b9f4a is 1080, key is row0097/info:/1733948113546/Put/seqid=0
2024-12-11T20:15:13,566 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35725 is added to blk_1073741856_1032 (size=12516)
2024-12-11T20:15:13,566 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741856_1032 (size=12516)
2024-12-11T20:15:13,569 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/.tmp/info/1255ca8191fa4d6a84e31726bb9b9f4a
2024-12-11T20:15:13,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/.tmp/info/1255ca8191fa4d6a84e31726bb9b9f4a as hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/1255ca8191fa4d6a84e31726bb9b9f4a
2024-12-11T20:15:13,581 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/1255ca8191fa4d6a84e31726bb9b9f4a, entries=7, sequenceid=137, filesize=12.2 K
2024-12-11T20:15:13,582 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=16.81 KB/17216 for 0000cbe68f7bb76d4869b7889b4f2fec in 26ms, sequenceid=137, compaction requested=false
2024-12-11T20:15:13,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0000cbe68f7bb76d4869b7889b4f2fec:

2024-12-11T20:15:13,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32831 {}] regionserver.HRegion(8855): Flush requested on 0000cbe68f7bb76d4869b7889b4f2fec
2024-12-11T20:15:13,582 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0000cbe68f7bb76d4869b7889b4f2fec 1/1 column families, dataSize=17.86 KB heapSize=19.38 KB
2024-12-11T20:15:13,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/.tmp/info/362597a506594b13b9aa100fef0f41d5 is 1080, key is row0104/info:/1733948113557/Put/seqid=0
2024-12-11T20:15:13,591 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35725 is added to blk_1073741857_1033 (size=23316)
2024-12-11T20:15:13,591 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741857_1033 (size=23316)
2024-12-11T20:15:13,592 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.86 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/.tmp/info/362597a506594b13b9aa100fef0f41d5
2024-12-11T20:15:13,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/.tmp/info/362597a506594b13b9aa100fef0f41d5 as hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/362597a506594b13b9aa100fef0f41d5
2024-12-11T20:15:13,602 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/362597a506594b13b9aa100fef0f41d5, entries=17, sequenceid=157, filesize=22.8 K
2024-12-11T20:15:13,603 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~17.86 KB/18292, heapSize ~19.36 KB/19824, currentSize=8.41 KB/8608 for 0000cbe68f7bb76d4869b7889b4f2fec in 21ms, sequenceid=157, compaction requested=true
2024-12-11T20:15:13,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0000cbe68f7bb76d4869b7889b4f2fec:

2024-12-11T20:15:13,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0000cbe68f7bb76d4869b7889b4f2fec:info, priority=-2147483648, current under compaction store size is 1
2024-12-11T20:15:13,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-11T20:15:13,603 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-11T20:15:13,604 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 78913 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-11T20:15:13,604 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.HStore(1541): 0000cbe68f7bb76d4869b7889b4f2fec/info is initiating minor compaction (all files)
2024-12-11T20:15:13,604 INFO  [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0000cbe68f7bb76d4869b7889b4f2fec/info in TestLogRolling-testLogRolling,row0062,1733948101484.0000cbe68f7bb76d4869b7889b4f2fec.
2024-12-11T20:15:13,605 INFO  [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/ca10a55495df44b8ae8e3e09c5b6eac4, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/1255ca8191fa4d6a84e31726bb9b9f4a, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/362597a506594b13b9aa100fef0f41d5] into tmpdir=hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/.tmp, totalSize=77.1 K
2024-12-11T20:15:13,605 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca10a55495df44b8ae8e3e09c5b6eac4, keycount=35, bloomtype=ROW, size=42.1 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1733948099407
2024-12-11T20:15:13,605 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1255ca8191fa4d6a84e31726bb9b9f4a, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1733948113546
2024-12-11T20:15:13,606 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 362597a506594b13b9aa100fef0f41d5, keycount=17, bloomtype=ROW, size=22.8 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733948113557
2024-12-11T20:15:13,616 INFO  [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0000cbe68f7bb76d4869b7889b4f2fec#info#compaction#73 average throughput is 60.54 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-11T20:15:13,617 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/.tmp/info/c36ed10eac94441f90a1b62154e25b44 is 1080, key is row0062/info:/1733948099407/Put/seqid=0
2024-12-11T20:15:13,639 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35725 is added to blk_1073741858_1034 (size=69123)
2024-12-11T20:15:13,639 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741858_1034 (size=69123)
2024-12-11T20:15:13,646 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/.tmp/info/c36ed10eac94441f90a1b62154e25b44 as hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/c36ed10eac94441f90a1b62154e25b44
2024-12-11T20:15:13,653 INFO  [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0000cbe68f7bb76d4869b7889b4f2fec/info of 0000cbe68f7bb76d4869b7889b4f2fec into c36ed10eac94441f90a1b62154e25b44(size=67.5 K), total size for store is 67.5 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-11T20:15:13,653 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0000cbe68f7bb76d4869b7889b4f2fec:

2024-12-11T20:15:13,653 INFO  [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733948101484.0000cbe68f7bb76d4869b7889b4f2fec., storeName=0000cbe68f7bb76d4869b7889b4f2fec/info, priority=13, startTime=1733948113603; duration=0sec
2024-12-11T20:15:13,653 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-11T20:15:13,653 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0000cbe68f7bb76d4869b7889b4f2fec:info
2024-12-11T20:15:14,345 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:15:14,345 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:15:15,346 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:15:15,346 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:15:15,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32831 {}] regionserver.HRegion(8855): Flush requested on 0000cbe68f7bb76d4869b7889b4f2fec
2024-12-11T20:15:15,599 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0000cbe68f7bb76d4869b7889b4f2fec 1/1 column families, dataSize=9.46 KB heapSize=10.38 KB
2024-12-11T20:15:15,603 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/.tmp/info/1dff70c802b54673b1711ee812cc60de is 1080, key is row0121/info:/1733948113583/Put/seqid=0
2024-12-11T20:15:15,608 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741859_1035 (size=14672)
2024-12-11T20:15:15,608 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35725 is added to blk_1073741859_1035 (size=14672)
2024-12-11T20:15:15,609 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=9.46 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/.tmp/info/1dff70c802b54673b1711ee812cc60de
2024-12-11T20:15:15,614 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/.tmp/info/1dff70c802b54673b1711ee812cc60de as hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/1dff70c802b54673b1711ee812cc60de
2024-12-11T20:15:15,620 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/1dff70c802b54673b1711ee812cc60de, entries=9, sequenceid=170, filesize=14.3 K
2024-12-11T20:15:15,621 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~9.46 KB/9684, heapSize ~10.36 KB/10608, currentSize=13.66 KB/13988 for 0000cbe68f7bb76d4869b7889b4f2fec in 23ms, sequenceid=170, compaction requested=false
2024-12-11T20:15:15,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0000cbe68f7bb76d4869b7889b4f2fec:

2024-12-11T20:15:15,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32831 {}] regionserver.HRegion(8855): Flush requested on 0000cbe68f7bb76d4869b7889b4f2fec
2024-12-11T20:15:15,622 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0000cbe68f7bb76d4869b7889b4f2fec 1/1 column families, dataSize=14.71 KB heapSize=16 KB
2024-12-11T20:15:15,627 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/.tmp/info/6dcf3328e72647ba94a92b4cfc06a1fb is 1080, key is row0130/info:/1733948115600/Put/seqid=0
2024-12-11T20:15:15,632 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35725 is added to blk_1073741860_1036 (size=20078)
2024-12-11T20:15:15,632 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741860_1036 (size=20078)
2024-12-11T20:15:15,632 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.71 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/.tmp/info/6dcf3328e72647ba94a92b4cfc06a1fb
2024-12-11T20:15:15,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/.tmp/info/6dcf3328e72647ba94a92b4cfc06a1fb as hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/6dcf3328e72647ba94a92b4cfc06a1fb
2024-12-11T20:15:15,644 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/6dcf3328e72647ba94a92b4cfc06a1fb, entries=14, sequenceid=187, filesize=19.6 K
2024-12-11T20:15:15,644 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~14.71 KB/15064, heapSize ~15.98 KB/16368, currentSize=14.71 KB/15064 for 0000cbe68f7bb76d4869b7889b4f2fec in 22ms, sequenceid=187, compaction requested=true
2024-12-11T20:15:15,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0000cbe68f7bb76d4869b7889b4f2fec:

2024-12-11T20:15:15,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0000cbe68f7bb76d4869b7889b4f2fec:info, priority=-2147483648, current under compaction store size is 1
2024-12-11T20:15:15,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-11T20:15:15,645 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-11T20:15:15,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32831 {}] regionserver.HRegion(8855): Flush requested on 0000cbe68f7bb76d4869b7889b4f2fec
2024-12-11T20:15:15,646 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0000cbe68f7bb76d4869b7889b4f2fec 1/1 column families, dataSize=15.76 KB heapSize=17.13 KB
2024-12-11T20:15:15,646 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 103873 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-11T20:15:15,646 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.HStore(1541): 0000cbe68f7bb76d4869b7889b4f2fec/info is initiating minor compaction (all files)
2024-12-11T20:15:15,646 INFO  [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0000cbe68f7bb76d4869b7889b4f2fec/info in TestLogRolling-testLogRolling,row0062,1733948101484.0000cbe68f7bb76d4869b7889b4f2fec.
2024-12-11T20:15:15,646 INFO  [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/c36ed10eac94441f90a1b62154e25b44, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/1dff70c802b54673b1711ee812cc60de, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/6dcf3328e72647ba94a92b4cfc06a1fb] into tmpdir=hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/.tmp, totalSize=101.4 K
2024-12-11T20:15:15,647 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] compactions.Compactor(225): Compacting c36ed10eac94441f90a1b62154e25b44, keycount=59, bloomtype=ROW, size=67.5 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733948099407
2024-12-11T20:15:15,647 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1dff70c802b54673b1711ee812cc60de, keycount=9, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733948113583
2024-12-11T20:15:15,648 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6dcf3328e72647ba94a92b4cfc06a1fb, keycount=14, bloomtype=ROW, size=19.6 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1733948115600
2024-12-11T20:15:15,650 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/.tmp/info/143c325d0c9148739aa8eaf1dbc87c54 is 1080, key is row0144/info:/1733948115623/Put/seqid=0
2024-12-11T20:15:15,656 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35725 is added to blk_1073741861_1037 (size=21156)
2024-12-11T20:15:15,656 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741861_1037 (size=21156)
2024-12-11T20:15:15,657 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.76 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/.tmp/info/143c325d0c9148739aa8eaf1dbc87c54
2024-12-11T20:15:15,660 INFO  [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0000cbe68f7bb76d4869b7889b4f2fec#info#compaction#77 average throughput is 42.07 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-11T20:15:15,661 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/.tmp/info/4e814ea598554bc7a7c42886d10f3289 is 1080, key is row0062/info:/1733948099407/Put/seqid=0
2024-12-11T20:15:15,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/.tmp/info/143c325d0c9148739aa8eaf1dbc87c54 as hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/143c325d0c9148739aa8eaf1dbc87c54
2024-12-11T20:15:15,666 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741862_1038 (size=94096)
2024-12-11T20:15:15,666 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35725 is added to blk_1073741862_1038 (size=94096)
2024-12-11T20:15:15,667 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/143c325d0c9148739aa8eaf1dbc87c54, entries=15, sequenceid=205, filesize=20.7 K
2024-12-11T20:15:15,668 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~15.76 KB/16140, heapSize ~17.11 KB/17520, currentSize=2.10 KB/2152 for 0000cbe68f7bb76d4869b7889b4f2fec in 22ms, sequenceid=205, compaction requested=false
2024-12-11T20:15:15,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0000cbe68f7bb76d4869b7889b4f2fec:

2024-12-11T20:15:15,671 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/.tmp/info/4e814ea598554bc7a7c42886d10f3289 as hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/4e814ea598554bc7a7c42886d10f3289
2024-12-11T20:15:15,676 INFO  [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0000cbe68f7bb76d4869b7889b4f2fec/info of 0000cbe68f7bb76d4869b7889b4f2fec into 4e814ea598554bc7a7c42886d10f3289(size=91.9 K), total size for store is 112.6 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-11T20:15:15,676 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0000cbe68f7bb76d4869b7889b4f2fec:

2024-12-11T20:15:15,676 INFO  [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733948101484.0000cbe68f7bb76d4869b7889b4f2fec., storeName=0000cbe68f7bb76d4869b7889b4f2fec/info, priority=13, startTime=1733948115645; duration=0sec
2024-12-11T20:15:15,676 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-11T20:15:15,676 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0000cbe68f7bb76d4869b7889b4f2fec:info
2024-12-11T20:15:16,346 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:15:16,346 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:15:17,347 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:15:17,347 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:15:17,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32831 {}] regionserver.HRegion(8855): Flush requested on 0000cbe68f7bb76d4869b7889b4f2fec
2024-12-11T20:15:17,658 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0000cbe68f7bb76d4869b7889b4f2fec 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-11T20:15:17,662 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/.tmp/info/337948c708794290981dc05eb50b9aa1 is 1080, key is row0159/info:/1733948115647/Put/seqid=0
2024-12-11T20:15:17,667 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35725 is added to blk_1073741863_1039 (size=12516)
2024-12-11T20:15:17,668 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741863_1039 (size=12516)
2024-12-11T20:15:17,668 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/.tmp/info/337948c708794290981dc05eb50b9aa1
2024-12-11T20:15:17,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/.tmp/info/337948c708794290981dc05eb50b9aa1 as hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/337948c708794290981dc05eb50b9aa1
2024-12-11T20:15:17,679 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/337948c708794290981dc05eb50b9aa1, entries=7, sequenceid=216, filesize=12.2 K
2024-12-11T20:15:17,680 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=14.71 KB/15064 for 0000cbe68f7bb76d4869b7889b4f2fec in 21ms, sequenceid=216, compaction requested=true
2024-12-11T20:15:17,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0000cbe68f7bb76d4869b7889b4f2fec:

2024-12-11T20:15:17,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0000cbe68f7bb76d4869b7889b4f2fec:info, priority=-2147483648, current under compaction store size is 1
2024-12-11T20:15:17,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-11T20:15:17,680 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-11T20:15:17,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32831 {}] regionserver.HRegion(8855): Flush requested on 0000cbe68f7bb76d4869b7889b4f2fec
2024-12-11T20:15:17,681 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0000cbe68f7bb76d4869b7889b4f2fec 1/1 column families, dataSize=15.76 KB heapSize=17.13 KB
2024-12-11T20:15:17,681 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 127768 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-11T20:15:17,681 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.HStore(1541): 0000cbe68f7bb76d4869b7889b4f2fec/info is initiating minor compaction (all files)
2024-12-11T20:15:17,681 INFO  [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0000cbe68f7bb76d4869b7889b4f2fec/info in TestLogRolling-testLogRolling,row0062,1733948101484.0000cbe68f7bb76d4869b7889b4f2fec.
2024-12-11T20:15:17,681 INFO  [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/4e814ea598554bc7a7c42886d10f3289, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/143c325d0c9148739aa8eaf1dbc87c54, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/337948c708794290981dc05eb50b9aa1] into tmpdir=hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/.tmp, totalSize=124.8 K
2024-12-11T20:15:17,682 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e814ea598554bc7a7c42886d10f3289, keycount=82, bloomtype=ROW, size=91.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1733948099407
2024-12-11T20:15:17,682 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 143c325d0c9148739aa8eaf1dbc87c54, keycount=15, bloomtype=ROW, size=20.7 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1733948115623
2024-12-11T20:15:17,682 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 337948c708794290981dc05eb50b9aa1, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1733948115647
2024-12-11T20:15:17,685 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/.tmp/info/48a114bdaac74bcda543428c6f1ab676 is 1080, key is row0166/info:/1733948117659/Put/seqid=0
2024-12-11T20:15:17,692 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35725 is added to blk_1073741864_1040 (size=21156)
2024-12-11T20:15:17,692 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741864_1040 (size=21156)
2024-12-11T20:15:17,692 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.76 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/.tmp/info/48a114bdaac74bcda543428c6f1ab676
2024-12-11T20:15:17,695 INFO  [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0000cbe68f7bb76d4869b7889b4f2fec#info#compaction#80 average throughput is 53.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-11T20:15:17,696 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/.tmp/info/1e30466cfd5441e786673b97a132e25f is 1080, key is row0062/info:/1733948099407/Put/seqid=0
2024-12-11T20:15:17,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/.tmp/info/48a114bdaac74bcda543428c6f1ab676 as hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/48a114bdaac74bcda543428c6f1ab676
2024-12-11T20:15:17,701 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35725 is added to blk_1073741865_1041 (size=117918)
2024-12-11T20:15:17,701 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741865_1041 (size=117918)
2024-12-11T20:15:17,704 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/48a114bdaac74bcda543428c6f1ab676, entries=15, sequenceid=234, filesize=20.7 K
2024-12-11T20:15:17,705 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~15.76 KB/16140, heapSize ~17.11 KB/17520, currentSize=12.61 KB/12912 for 0000cbe68f7bb76d4869b7889b4f2fec in 24ms, sequenceid=234, compaction requested=false
2024-12-11T20:15:17,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0000cbe68f7bb76d4869b7889b4f2fec:

2024-12-11T20:15:17,707 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/.tmp/info/1e30466cfd5441e786673b97a132e25f as hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/1e30466cfd5441e786673b97a132e25f
2024-12-11T20:15:17,712 INFO  [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0000cbe68f7bb76d4869b7889b4f2fec/info of 0000cbe68f7bb76d4869b7889b4f2fec into 1e30466cfd5441e786673b97a132e25f(size=115.2 K), total size for store is 135.8 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-11T20:15:17,712 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0000cbe68f7bb76d4869b7889b4f2fec:

2024-12-11T20:15:17,712 INFO  [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733948101484.0000cbe68f7bb76d4869b7889b4f2fec., storeName=0000cbe68f7bb76d4869b7889b4f2fec/info, priority=13, startTime=1733948117680; duration=0sec
2024-12-11T20:15:17,712 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-11T20:15:17,712 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0000cbe68f7bb76d4869b7889b4f2fec:info
2024-12-11T20:15:17,837 INFO  [master/9fe0640122ec:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-12-11T20:15:17,837 INFO  [master/9fe0640122ec:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-12-11T20:15:18,347 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:15:18,347 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:15:19,348 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:15:19,348 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:15:19,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32831 {}] regionserver.HRegion(8855): Flush requested on 0000cbe68f7bb76d4869b7889b4f2fec
2024-12-11T20:15:19,707 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0000cbe68f7bb76d4869b7889b4f2fec 1/1 column families, dataSize=13.66 KB heapSize=14.88 KB
2024-12-11T20:15:19,711 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/.tmp/info/ca5a0e82c30c4ee2b2163140d22def1f is 1080, key is row0181/info:/1733948117682/Put/seqid=0
2024-12-11T20:15:19,716 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741866_1042 (size=19000)
2024-12-11T20:15:19,717 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35725 is added to blk_1073741866_1042 (size=19000)
2024-12-11T20:15:19,717 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.66 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/.tmp/info/ca5a0e82c30c4ee2b2163140d22def1f
2024-12-11T20:15:19,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/.tmp/info/ca5a0e82c30c4ee2b2163140d22def1f as hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/ca5a0e82c30c4ee2b2163140d22def1f
2024-12-11T20:15:19,728 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/ca5a0e82c30c4ee2b2163140d22def1f, entries=13, sequenceid=251, filesize=18.6 K
2024-12-11T20:15:19,729 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.66 KB/13988, heapSize ~14.86 KB/15216, currentSize=13.66 KB/13988 for 0000cbe68f7bb76d4869b7889b4f2fec in 21ms, sequenceid=251, compaction requested=true
2024-12-11T20:15:19,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0000cbe68f7bb76d4869b7889b4f2fec:

2024-12-11T20:15:19,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0000cbe68f7bb76d4869b7889b4f2fec:info, priority=-2147483648, current under compaction store size is 1
2024-12-11T20:15:19,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-11T20:15:19,729 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-11T20:15:19,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32831 {}] regionserver.HRegion(8855): Flush requested on 0000cbe68f7bb76d4869b7889b4f2fec
2024-12-11T20:15:19,730 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 158074 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-11T20:15:19,730 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.HStore(1541): 0000cbe68f7bb76d4869b7889b4f2fec/info is initiating minor compaction (all files)
2024-12-11T20:15:19,730 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0000cbe68f7bb76d4869b7889b4f2fec 1/1 column families, dataSize=14.71 KB heapSize=16 KB
2024-12-11T20:15:19,730 INFO  [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0000cbe68f7bb76d4869b7889b4f2fec/info in TestLogRolling-testLogRolling,row0062,1733948101484.0000cbe68f7bb76d4869b7889b4f2fec.
2024-12-11T20:15:19,730 INFO  [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/1e30466cfd5441e786673b97a132e25f, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/48a114bdaac74bcda543428c6f1ab676, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/ca5a0e82c30c4ee2b2163140d22def1f] into tmpdir=hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/.tmp, totalSize=154.4 K
2024-12-11T20:15:19,731 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e30466cfd5441e786673b97a132e25f, keycount=104, bloomtype=ROW, size=115.2 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1733948099407
2024-12-11T20:15:19,731 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48a114bdaac74bcda543428c6f1ab676, keycount=15, bloomtype=ROW, size=20.7 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1733948117659
2024-12-11T20:15:19,732 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca5a0e82c30c4ee2b2163140d22def1f, keycount=13, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1733948117682
2024-12-11T20:15:19,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/.tmp/info/d93ca67ac0cc4571821be733c24e570e is 1080, key is row0194/info:/1733948119708/Put/seqid=0
2024-12-11T20:15:19,742 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741867_1043 (size=20089)
2024-12-11T20:15:19,742 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35725 is added to blk_1073741867_1043 (size=20089)
2024-12-11T20:15:19,742 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.71 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/.tmp/info/d93ca67ac0cc4571821be733c24e570e
2024-12-11T20:15:19,744 INFO  [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0000cbe68f7bb76d4869b7889b4f2fec#info#compaction#83 average throughput is 45.15 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-11T20:15:19,745 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/.tmp/info/29e0d48e60d245b0ac183f423ad351df is 1080, key is row0062/info:/1733948099407/Put/seqid=0
2024-12-11T20:15:19,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/.tmp/info/d93ca67ac0cc4571821be733c24e570e as hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/d93ca67ac0cc4571821be733c24e570e
2024-12-11T20:15:19,749 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35725 is added to blk_1073741868_1044 (size=148409)
2024-12-11T20:15:19,750 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741868_1044 (size=148409)
2024-12-11T20:15:19,755 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/d93ca67ac0cc4571821be733c24e570e, entries=14, sequenceid=268, filesize=19.6 K
2024-12-11T20:15:19,755 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/.tmp/info/29e0d48e60d245b0ac183f423ad351df as hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/29e0d48e60d245b0ac183f423ad351df
2024-12-11T20:15:19,756 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~14.71 KB/15064, heapSize ~15.98 KB/16368, currentSize=14.71 KB/15064 for 0000cbe68f7bb76d4869b7889b4f2fec in 26ms, sequenceid=268, compaction requested=false
2024-12-11T20:15:19,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0000cbe68f7bb76d4869b7889b4f2fec:

2024-12-11T20:15:19,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32831 {}] regionserver.HRegion(8855): Flush requested on 0000cbe68f7bb76d4869b7889b4f2fec
2024-12-11T20:15:19,757 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0000cbe68f7bb76d4869b7889b4f2fec 1/1 column families, dataSize=15.76 KB heapSize=17.13 KB
2024-12-11T20:15:19,761 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/.tmp/info/5528e25e9dee41d19d85262022358986 is 1080, key is row0208/info:/1733948119731/Put/seqid=0
2024-12-11T20:15:19,762 INFO  [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0000cbe68f7bb76d4869b7889b4f2fec/info of 0000cbe68f7bb76d4869b7889b4f2fec into 29e0d48e60d245b0ac183f423ad351df(size=144.9 K), total size for store is 164.5 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-11T20:15:19,762 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0000cbe68f7bb76d4869b7889b4f2fec:

2024-12-11T20:15:19,763 INFO  [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733948101484.0000cbe68f7bb76d4869b7889b4f2fec., storeName=0000cbe68f7bb76d4869b7889b4f2fec/info, priority=13, startTime=1733948119729; duration=0sec
2024-12-11T20:15:19,763 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-11T20:15:19,763 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0000cbe68f7bb76d4869b7889b4f2fec:info
2024-12-11T20:15:19,765 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741869_1045 (size=21171)
2024-12-11T20:15:19,766 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35725 is added to blk_1073741869_1045 (size=21171)
2024-12-11T20:15:19,767 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.76 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/.tmp/info/5528e25e9dee41d19d85262022358986
2024-12-11T20:15:19,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/.tmp/info/5528e25e9dee41d19d85262022358986 as hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/5528e25e9dee41d19d85262022358986
2024-12-11T20:15:19,776 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/5528e25e9dee41d19d85262022358986, entries=15, sequenceid=286, filesize=20.7 K
2024-12-11T20:15:19,777 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~15.76 KB/16140, heapSize ~17.11 KB/17520, currentSize=2.10 KB/2152 for 0000cbe68f7bb76d4869b7889b4f2fec in 20ms, sequenceid=286, compaction requested=true
2024-12-11T20:15:19,777 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0000cbe68f7bb76d4869b7889b4f2fec:

2024-12-11T20:15:19,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0000cbe68f7bb76d4869b7889b4f2fec:info, priority=-2147483648, current under compaction store size is 1
2024-12-11T20:15:19,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-11T20:15:19,777 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-11T20:15:19,778 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 189669 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-11T20:15:19,778 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.HStore(1541): 0000cbe68f7bb76d4869b7889b4f2fec/info is initiating minor compaction (all files)
2024-12-11T20:15:19,778 INFO  [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0000cbe68f7bb76d4869b7889b4f2fec/info in TestLogRolling-testLogRolling,row0062,1733948101484.0000cbe68f7bb76d4869b7889b4f2fec.
2024-12-11T20:15:19,778 INFO  [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/29e0d48e60d245b0ac183f423ad351df, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/d93ca67ac0cc4571821be733c24e570e, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/5528e25e9dee41d19d85262022358986] into tmpdir=hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/.tmp, totalSize=185.2 K
2024-12-11T20:15:19,779 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29e0d48e60d245b0ac183f423ad351df, keycount=132, bloomtype=ROW, size=144.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1733948099407
2024-12-11T20:15:19,779 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] compactions.Compactor(225): Compacting d93ca67ac0cc4571821be733c24e570e, keycount=14, bloomtype=ROW, size=19.6 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1733948119708
2024-12-11T20:15:19,779 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5528e25e9dee41d19d85262022358986, keycount=15, bloomtype=ROW, size=20.7 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733948119731
2024-12-11T20:15:19,789 INFO  [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0000cbe68f7bb76d4869b7889b4f2fec#info#compaction#85 average throughput is 82.61 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-11T20:15:19,790 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/.tmp/info/79a92d735df3439cacb0d17e2a1d1561 is 1080, key is row0062/info:/1733948099407/Put/seqid=0
2024-12-11T20:15:19,793 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35725 is added to blk_1073741870_1046 (size=179807)
2024-12-11T20:15:19,793 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741870_1046 (size=179807)
2024-12-11T20:15:19,798 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/.tmp/info/79a92d735df3439cacb0d17e2a1d1561 as hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/79a92d735df3439cacb0d17e2a1d1561
2024-12-11T20:15:19,803 INFO  [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0000cbe68f7bb76d4869b7889b4f2fec/info of 0000cbe68f7bb76d4869b7889b4f2fec into 79a92d735df3439cacb0d17e2a1d1561(size=175.6 K), total size for store is 175.6 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-11T20:15:19,803 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0000cbe68f7bb76d4869b7889b4f2fec:

2024-12-11T20:15:19,803 INFO  [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733948101484.0000cbe68f7bb76d4869b7889b4f2fec., storeName=0000cbe68f7bb76d4869b7889b4f2fec/info, priority=13, startTime=1733948119777; duration=0sec
2024-12-11T20:15:19,803 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-11T20:15:19,803 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0000cbe68f7bb76d4869b7889b4f2fec:info
2024-12-11T20:15:20,349 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:15:20,349 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:15:21,349 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:15:21,349 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:15:21,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32831 {}] regionserver.HRegion(8855): Flush requested on 0000cbe68f7bb76d4869b7889b4f2fec
2024-12-11T20:15:21,768 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0000cbe68f7bb76d4869b7889b4f2fec 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-11T20:15:21,773 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/.tmp/info/490c0afab5e94da68f92a6047758464f is 1080, key is row0223/info:/1733948119758/Put/seqid=0
2024-12-11T20:15:21,778 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741871_1047 (size=12523)
2024-12-11T20:15:21,778 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35725 is added to blk_1073741871_1047 (size=12523)
2024-12-11T20:15:21,779 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/.tmp/info/490c0afab5e94da68f92a6047758464f
2024-12-11T20:15:21,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/.tmp/info/490c0afab5e94da68f92a6047758464f as hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/490c0afab5e94da68f92a6047758464f
2024-12-11T20:15:21,790 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/490c0afab5e94da68f92a6047758464f, entries=7, sequenceid=298, filesize=12.2 K
2024-12-11T20:15:21,791 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=13.66 KB/13988 for 0000cbe68f7bb76d4869b7889b4f2fec in 23ms, sequenceid=298, compaction requested=false
2024-12-11T20:15:21,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0000cbe68f7bb76d4869b7889b4f2fec:

2024-12-11T20:15:21,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32831 {}] regionserver.HRegion(8855): Flush requested on 0000cbe68f7bb76d4869b7889b4f2fec
2024-12-11T20:15:21,791 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0000cbe68f7bb76d4869b7889b4f2fec 1/1 column families, dataSize=14.71 KB heapSize=16 KB
2024-12-11T20:15:21,795 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/.tmp/info/9a334998d99f44c18766b14627765ed7 is 1080, key is row0230/info:/1733948121769/Put/seqid=0
2024-12-11T20:15:21,800 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35725 is added to blk_1073741872_1048 (size=20092)
2024-12-11T20:15:21,801 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741872_1048 (size=20092)
2024-12-11T20:15:21,801 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.71 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/.tmp/info/9a334998d99f44c18766b14627765ed7
2024-12-11T20:15:21,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/.tmp/info/9a334998d99f44c18766b14627765ed7 as hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/9a334998d99f44c18766b14627765ed7
2024-12-11T20:15:21,811 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/9a334998d99f44c18766b14627765ed7, entries=14, sequenceid=315, filesize=19.6 K
2024-12-11T20:15:21,812 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~14.71 KB/15064, heapSize ~15.98 KB/16368, currentSize=13.66 KB/13988 for 0000cbe68f7bb76d4869b7889b4f2fec in 21ms, sequenceid=315, compaction requested=true
2024-12-11T20:15:21,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0000cbe68f7bb76d4869b7889b4f2fec:

2024-12-11T20:15:21,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0000cbe68f7bb76d4869b7889b4f2fec:info, priority=-2147483648, current under compaction store size is 1
2024-12-11T20:15:21,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-11T20:15:21,812 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-11T20:15:21,813 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 212422 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-11T20:15:21,813 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.HStore(1541): 0000cbe68f7bb76d4869b7889b4f2fec/info is initiating minor compaction (all files)
2024-12-11T20:15:21,813 INFO  [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0000cbe68f7bb76d4869b7889b4f2fec/info in TestLogRolling-testLogRolling,row0062,1733948101484.0000cbe68f7bb76d4869b7889b4f2fec.
2024-12-11T20:15:21,813 INFO  [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/79a92d735df3439cacb0d17e2a1d1561, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/490c0afab5e94da68f92a6047758464f, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/9a334998d99f44c18766b14627765ed7] into tmpdir=hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/.tmp, totalSize=207.4 K
2024-12-11T20:15:21,814 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79a92d735df3439cacb0d17e2a1d1561, keycount=161, bloomtype=ROW, size=175.6 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733948099407
2024-12-11T20:15:21,814 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 490c0afab5e94da68f92a6047758464f, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1733948119758
2024-12-11T20:15:21,814 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a334998d99f44c18766b14627765ed7, keycount=14, bloomtype=ROW, size=19.6 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1733948121769
2024-12-11T20:15:21,826 INFO  [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0000cbe68f7bb76d4869b7889b4f2fec#info#compaction#88 average throughput is 93.38 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-11T20:15:21,827 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/.tmp/info/e0277192e8744c63a175d4b7235de726 is 1080, key is row0062/info:/1733948099407/Put/seqid=0
2024-12-11T20:15:21,830 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741873_1049 (size=202572)
2024-12-11T20:15:21,830 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35725 is added to blk_1073741873_1049 (size=202572)
2024-12-11T20:15:21,836 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/.tmp/info/e0277192e8744c63a175d4b7235de726 as hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/e0277192e8744c63a175d4b7235de726
2024-12-11T20:15:21,841 INFO  [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0000cbe68f7bb76d4869b7889b4f2fec/info of 0000cbe68f7bb76d4869b7889b4f2fec into e0277192e8744c63a175d4b7235de726(size=197.8 K), total size for store is 197.8 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-11T20:15:21,842 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0000cbe68f7bb76d4869b7889b4f2fec:

2024-12-11T20:15:21,842 INFO  [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733948101484.0000cbe68f7bb76d4869b7889b4f2fec., storeName=0000cbe68f7bb76d4869b7889b4f2fec/info, priority=13, startTime=1733948121812; duration=0sec
2024-12-11T20:15:21,842 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-11T20:15:21,842 DEBUG [RS:0;9fe0640122ec:32831-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0000cbe68f7bb76d4869b7889b4f2fec:info
2024-12-11T20:15:22,075 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 20375
2024-12-11T20:15:22,350 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:15:22,350 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:15:23,350 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:15:23,350 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:15:23,810 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(285): after writing there are 0 log files
2024-12-11T20:15:23,811 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9fe0640122ec%2C32831%2C1733948076323.1733948123811
2024-12-11T20:15:23,817 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:15:23,817 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:15:23,817 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:15:23,817 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:15:23,817 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:15:23,817 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/WALs/9fe0640122ec,32831,1733948076323/9fe0640122ec%2C32831%2C1733948076323.1733948076708 with entries=310, filesize=307.89 KB; new WAL /user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/WALs/9fe0640122ec,32831,1733948076323/9fe0640122ec%2C32831%2C1733948076323.1733948123811
2024-12-11T20:15:23,818 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33307:33307),(127.0.0.1/127.0.0.1:39125:39125)]
2024-12-11T20:15:23,818 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/WALs/9fe0640122ec,32831,1733948076323/9fe0640122ec%2C32831%2C1733948076323.1733948076708 is not closed yet, will try archiving it next time
2024-12-11T20:15:23,819 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35725 is added to blk_1073741833_1009 (size=315283)
2024-12-11T20:15:23,819 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741833_1009 (size=315283)
2024-12-11T20:15:23,822 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 0000cbe68f7bb76d4869b7889b4f2fec 1/1 column families, dataSize=13.66 KB heapSize=14.88 KB
2024-12-11T20:15:23,826 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/.tmp/info/f1abd15a0d3f416fbb46dbf0e281ee3f is 1080, key is row0244/info:/1733948121792/Put/seqid=0
2024-12-11T20:15:23,830 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741875_1051 (size=19013)
2024-12-11T20:15:23,831 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35725 is added to blk_1073741875_1051 (size=19013)
2024-12-11T20:15:23,831 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.66 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/.tmp/info/f1abd15a0d3f416fbb46dbf0e281ee3f
2024-12-11T20:15:23,836 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/.tmp/info/f1abd15a0d3f416fbb46dbf0e281ee3f as hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/f1abd15a0d3f416fbb46dbf0e281ee3f
2024-12-11T20:15:23,841 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/f1abd15a0d3f416fbb46dbf0e281ee3f, entries=13, sequenceid=332, filesize=18.6 K
2024-12-11T20:15:23,842 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.66 KB/13988, heapSize ~14.86 KB/15216, currentSize=0 B/0 for 0000cbe68f7bb76d4869b7889b4f2fec in 20ms, sequenceid=332, compaction requested=false
2024-12-11T20:15:23,842 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 0000cbe68f7bb76d4869b7889b4f2fec:

2024-12-11T20:15:23,842 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 607a8af8bf8fceec868c0a5c1d5fefa1:

2024-12-11T20:15:23,842 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=670 B heapSize=2.02 KB
2024-12-11T20:15:23,846 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/hbase/meta/1588230740/.tmp/info/0da216cdb8524559bd82a0d306216787 is 186, key is TestLogRolling-testLogRolling,,1733948101484.607a8af8bf8fceec868c0a5c1d5fefa1./info:regioninfo/1733948102187/Put/seqid=0
2024-12-11T20:15:23,850 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741876_1052 (size=6153)
2024-12-11T20:15:23,850 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35725 is added to blk_1073741876_1052 (size=6153)
2024-12-11T20:15:23,850 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=670 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/hbase/meta/1588230740/.tmp/info/0da216cdb8524559bd82a0d306216787
2024-12-11T20:15:23,855 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/hbase/meta/1588230740/.tmp/info/0da216cdb8524559bd82a0d306216787 as hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/hbase/meta/1588230740/info/0da216cdb8524559bd82a0d306216787
2024-12-11T20:15:23,860 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/hbase/meta/1588230740/info/0da216cdb8524559bd82a0d306216787, entries=5, sequenceid=21, filesize=6.0 K
2024-12-11T20:15:23,861 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~670 B/670, heapSize ~1.25 KB/1280, currentSize=0 B/0 for 1588230740 in 19ms, sequenceid=21, compaction requested=false
2024-12-11T20:15:23,861 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1588230740:

2024-12-11T20:15:23,861 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9fe0640122ec%2C32831%2C1733948076323.1733948123861
2024-12-11T20:15:23,865 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:15:23,865 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:15:23,865 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:15:23,865 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:15:23,866 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:15:23,866 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/WALs/9fe0640122ec,32831,1733948076323/9fe0640122ec%2C32831%2C1733948076323.1733948123811 with entries=2, filesize=723 B; new WAL /user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/WALs/9fe0640122ec,32831,1733948076323/9fe0640122ec%2C32831%2C1733948076323.1733948123861
2024-12-11T20:15:23,867 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33307:33307),(127.0.0.1/127.0.0.1:39125:39125)]
2024-12-11T20:15:23,867 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/WALs/9fe0640122ec,32831,1733948076323/9fe0640122ec%2C32831%2C1733948076323.1733948123811 is not closed yet, will try archiving it next time
2024-12-11T20:15:23,867 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35725 is added to blk_1073741874_1050 (size=731)
2024-12-11T20:15:23,867 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741874_1050 (size=731)
2024-12-11T20:15:23,869 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/WALs/9fe0640122ec,32831,1733948076323/9fe0640122ec%2C32831%2C1733948076323.1733948076708 to hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/oldWALs/9fe0640122ec%2C32831%2C1733948076323.1733948076708
2024-12-11T20:15:23,869 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1])
2024-12-11T20:15:23,869 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-12-11T20:15:23,869 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-11T20:15:23,869 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-11T20:15:23,870 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T20:15:23,870 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T20:15:23,870 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-12-11T20:15:23,870 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/WALs/9fe0640122ec,32831,1733948076323/9fe0640122ec%2C32831%2C1733948076323.1733948123811 to hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/oldWALs/9fe0640122ec%2C32831%2C1733948076323.1733948123811
2024-12-11T20:15:23,870 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-11T20:15:23,870 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=394603458, stopped=false
2024-12-11T20:15:23,870 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=9fe0640122ec,46569,1733948076281
2024-12-11T20:15:23,872 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32831-0x100cafa036a0001, quorum=127.0.0.1:53734, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-11T20:15:23,872 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32831-0x100cafa036a0001, quorum=127.0.0.1:53734, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:15:23,872 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46569-0x100cafa036a0000, quorum=127.0.0.1:53734, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-11T20:15:23,872 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-11T20:15:23,872 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46569-0x100cafa036a0000, quorum=127.0.0.1:53734, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:15:23,872 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-11T20:15:23,872 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-11T20:15:23,872 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T20:15:23,873 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:32831-0x100cafa036a0001, quorum=127.0.0.1:53734, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-11T20:15:23,873 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9fe0640122ec,32831,1733948076323' *****
2024-12-11T20:15:23,873 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-12-11T20:15:23,873 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:46569-0x100cafa036a0000, quorum=127.0.0.1:53734, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-11T20:15:23,873 INFO  [RS:0;9fe0640122ec:32831 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-11T20:15:23,873 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-12-11T20:15:23,873 INFO  [RS:0;9fe0640122ec:32831 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-11T20:15:23,873 INFO  [RS:0;9fe0640122ec:32831 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-11T20:15:23,874 INFO  [RS:0;9fe0640122ec:32831 {}] regionserver.HRegionServer(3091): Received CLOSE for 0000cbe68f7bb76d4869b7889b4f2fec
2024-12-11T20:15:23,874 INFO  [RS:0;9fe0640122ec:32831 {}] regionserver.HRegionServer(3091): Received CLOSE for 607a8af8bf8fceec868c0a5c1d5fefa1
2024-12-11T20:15:23,874 INFO  [RS:0;9fe0640122ec:32831 {}] regionserver.HRegionServer(959): stopping server 9fe0640122ec,32831,1733948076323
2024-12-11T20:15:23,874 INFO  [RS:0;9fe0640122ec:32831 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-11T20:15:23,874 INFO  [RS:0;9fe0640122ec:32831 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;9fe0640122ec:32831.
2024-12-11T20:15:23,874 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0000cbe68f7bb76d4869b7889b4f2fec, disabling compactions & flushes
2024-12-11T20:15:23,874 DEBUG [RS:0;9fe0640122ec:32831 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-11T20:15:23,874 DEBUG [RS:0;9fe0640122ec:32831 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T20:15:23,874 INFO  [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,row0062,1733948101484.0000cbe68f7bb76d4869b7889b4f2fec.
2024-12-11T20:15:23,874 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,row0062,1733948101484.0000cbe68f7bb76d4869b7889b4f2fec.
2024-12-11T20:15:23,874 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,row0062,1733948101484.0000cbe68f7bb76d4869b7889b4f2fec. after waiting 0 ms
2024-12-11T20:15:23,874 INFO  [RS:0;9fe0640122ec:32831 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-11T20:15:23,874 INFO  [RS:0;9fe0640122ec:32831 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-11T20:15:23,874 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,row0062,1733948101484.0000cbe68f7bb76d4869b7889b4f2fec.
2024-12-11T20:15:23,874 INFO  [RS:0;9fe0640122ec:32831 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-11T20:15:23,874 INFO  [RS:0;9fe0640122ec:32831 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-12-11T20:15:23,874 INFO  [RS:0;9fe0640122ec:32831 {}] regionserver.HRegionServer(1321): Waiting on 3 regions to close
2024-12-11T20:15:23,874 DEBUG [RS:0;9fe0640122ec:32831 {}] regionserver.HRegionServer(1325): Online Regions={0000cbe68f7bb76d4869b7889b4f2fec=TestLogRolling-testLogRolling,row0062,1733948101484.0000cbe68f7bb76d4869b7889b4f2fec., 607a8af8bf8fceec868c0a5c1d5fefa1=TestLogRolling-testLogRolling,,1733948101484.607a8af8bf8fceec868c0a5c1d5fefa1., 1588230740=hbase:meta,,1.1588230740}
2024-12-11T20:15:23,875 DEBUG [RS:0;9fe0640122ec:32831 {}] regionserver.HRegionServer(1351): Waiting on 0000cbe68f7bb76d4869b7889b4f2fec, 1588230740, 607a8af8bf8fceec868c0a5c1d5fefa1
2024-12-11T20:15:23,875 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-11T20:15:23,875 INFO  [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-11T20:15:23,875 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-11T20:15:23,875 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-11T20:15:23,875 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-11T20:15:23,874 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733948101484.0000cbe68f7bb76d4869b7889b4f2fec.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/588b5cca3d0c4abba65d01b91f92cc1b.07f5c44b5ba0e29fbf0ec02f39733158->hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/info/588b5cca3d0c4abba65d01b91f92cc1b-top, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/TestLogRolling-testLogRolling=07f5c44b5ba0e29fbf0ec02f39733158-686f9b3bdb70430b955f58147222b7c9, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/TestLogRolling-testLogRolling=07f5c44b5ba0e29fbf0ec02f39733158-cf43db9cd99c41e3bdad620556077e4b, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/ca10a55495df44b8ae8e3e09c5b6eac4, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/TestLogRolling-testLogRolling=07f5c44b5ba0e29fbf0ec02f39733158-f1b31f7a36a54d389928d5566883f385, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/1255ca8191fa4d6a84e31726bb9b9f4a, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/c36ed10eac94441f90a1b62154e25b44, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/362597a506594b13b9aa100fef0f41d5, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/1dff70c802b54673b1711ee812cc60de, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/4e814ea598554bc7a7c42886d10f3289, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/6dcf3328e72647ba94a92b4cfc06a1fb, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/143c325d0c9148739aa8eaf1dbc87c54, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/1e30466cfd5441e786673b97a132e25f, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/337948c708794290981dc05eb50b9aa1, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/48a114bdaac74bcda543428c6f1ab676, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/29e0d48e60d245b0ac183f423ad351df, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/ca5a0e82c30c4ee2b2163140d22def1f, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/d93ca67ac0cc4571821be733c24e570e, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/79a92d735df3439cacb0d17e2a1d1561, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/5528e25e9dee41d19d85262022358986, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/490c0afab5e94da68f92a6047758464f, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/9a334998d99f44c18766b14627765ed7] to archive
2024-12-11T20:15:23,875 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733948101484.0000cbe68f7bb76d4869b7889b4f2fec.-1 {}] backup.HFileArchiver(363): Archiving compacted files.
2024-12-11T20:15:23,878 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/588b5cca3d0c4abba65d01b91f92cc1b.07f5c44b5ba0e29fbf0ec02f39733158 to hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/archive/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/588b5cca3d0c4abba65d01b91f92cc1b.07f5c44b5ba0e29fbf0ec02f39733158
2024-12-11T20:15:23,878 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/TestLogRolling-testLogRolling=07f5c44b5ba0e29fbf0ec02f39733158-cf43db9cd99c41e3bdad620556077e4b to hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/archive/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/TestLogRolling-testLogRolling=07f5c44b5ba0e29fbf0ec02f39733158-cf43db9cd99c41e3bdad620556077e4b
2024-12-11T20:15:23,878 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/TestLogRolling-testLogRolling=07f5c44b5ba0e29fbf0ec02f39733158-686f9b3bdb70430b955f58147222b7c9 to hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/archive/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/TestLogRolling-testLogRolling=07f5c44b5ba0e29fbf0ec02f39733158-686f9b3bdb70430b955f58147222b7c9
2024-12-11T20:15:23,878 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/TestLogRolling-testLogRolling=07f5c44b5ba0e29fbf0ec02f39733158-f1b31f7a36a54d389928d5566883f385 to hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/archive/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/TestLogRolling-testLogRolling=07f5c44b5ba0e29fbf0ec02f39733158-f1b31f7a36a54d389928d5566883f385
2024-12-11T20:15:23,878 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/362597a506594b13b9aa100fef0f41d5 to hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/archive/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/362597a506594b13b9aa100fef0f41d5
2024-12-11T20:15:23,879 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/1255ca8191fa4d6a84e31726bb9b9f4a to hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/archive/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/1255ca8191fa4d6a84e31726bb9b9f4a
2024-12-11T20:15:23,879 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/ca10a55495df44b8ae8e3e09c5b6eac4 to hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/archive/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/ca10a55495df44b8ae8e3e09c5b6eac4
2024-12-11T20:15:23,880 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/4e814ea598554bc7a7c42886d10f3289 to hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/archive/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/4e814ea598554bc7a7c42886d10f3289
2024-12-11T20:15:23,880 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/6dcf3328e72647ba94a92b4cfc06a1fb to hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/archive/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/6dcf3328e72647ba94a92b4cfc06a1fb
2024-12-11T20:15:23,881 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/143c325d0c9148739aa8eaf1dbc87c54 to hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/archive/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/143c325d0c9148739aa8eaf1dbc87c54
2024-12-11T20:15:23,881 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/337948c708794290981dc05eb50b9aa1 to hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/archive/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/337948c708794290981dc05eb50b9aa1
2024-12-11T20:15:23,881 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/48a114bdaac74bcda543428c6f1ab676 to hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/archive/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/48a114bdaac74bcda543428c6f1ab676
2024-12-11T20:15:23,881 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/1e30466cfd5441e786673b97a132e25f to hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/archive/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/1e30466cfd5441e786673b97a132e25f
2024-12-11T20:15:23,881 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/c36ed10eac94441f90a1b62154e25b44 to hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/archive/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/c36ed10eac94441f90a1b62154e25b44
2024-12-11T20:15:23,881 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1
2024-12-11T20:15:23,881 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/1dff70c802b54673b1711ee812cc60de to hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/archive/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/1dff70c802b54673b1711ee812cc60de
2024-12-11T20:15:23,882 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-11T20:15:23,882 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/ca5a0e82c30c4ee2b2163140d22def1f to hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/archive/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/ca5a0e82c30c4ee2b2163140d22def1f
2024-12-11T20:15:23,883 INFO  [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-11T20:15:23,883 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733948123875Running coprocessor pre-close hooks at 1733948123875Disabling compacts and flushes for region at 1733948123875Disabling writes for close at 1733948123875Writing region close event to WAL at 1733948123876 (+1 ms)Running coprocessor post-close hooks at 1733948123882 (+6 ms)Closed at 1733948123883 (+1 ms)
2024-12-11T20:15:23,883 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/29e0d48e60d245b0ac183f423ad351df to hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/archive/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/29e0d48e60d245b0ac183f423ad351df
2024-12-11T20:15:23,883 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-11T20:15:23,883 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/79a92d735df3439cacb0d17e2a1d1561 to hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/archive/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/79a92d735df3439cacb0d17e2a1d1561
2024-12-11T20:15:23,883 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/d93ca67ac0cc4571821be733c24e570e to hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/archive/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/d93ca67ac0cc4571821be733c24e570e
2024-12-11T20:15:23,883 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/490c0afab5e94da68f92a6047758464f to hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/archive/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/490c0afab5e94da68f92a6047758464f
2024-12-11T20:15:23,883 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/5528e25e9dee41d19d85262022358986 to hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/archive/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/5528e25e9dee41d19d85262022358986
2024-12-11T20:15:23,884 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/9a334998d99f44c18766b14627765ed7 to hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/archive/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/info/9a334998d99f44c18766b14627765ed7
2024-12-11T20:15:23,884 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733948101484.0000cbe68f7bb76d4869b7889b4f2fec.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=9fe0640122ec:46569 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 16 more
2024-12-11T20:15:23,884 WARN  [StoreCloser-TestLogRolling-testLogRolling,row0062,1733948101484.0000cbe68f7bb76d4869b7889b4f2fec.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [ca10a55495df44b8ae8e3e09c5b6eac4=43081, 1255ca8191fa4d6a84e31726bb9b9f4a=12516, c36ed10eac94441f90a1b62154e25b44=69123, 362597a506594b13b9aa100fef0f41d5=23316, 1dff70c802b54673b1711ee812cc60de=14672, 4e814ea598554bc7a7c42886d10f3289=94096, 6dcf3328e72647ba94a92b4cfc06a1fb=20078, 143c325d0c9148739aa8eaf1dbc87c54=21156, 1e30466cfd5441e786673b97a132e25f=117918, 337948c708794290981dc05eb50b9aa1=12516, 48a114bdaac74bcda543428c6f1ab676=21156, 29e0d48e60d245b0ac183f423ad351df=148409, ca5a0e82c30c4ee2b2163140d22def1f=19000, d93ca67ac0cc4571821be733c24e570e=20089, 79a92d735df3439cacb0d17e2a1d1561=179807, 5528e25e9dee41d19d85262022358986=21171, 490c0afab5e94da68f92a6047758464f=12523, 9a334998d99f44c18766b14627765ed7=20092]
2024-12-11T20:15:23,887 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/0000cbe68f7bb76d4869b7889b4f2fec/recovered.edits/335.seqid, newMaxSeqId=335, maxSeqId=126
2024-12-11T20:15:23,888 INFO  [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,row0062,1733948101484.0000cbe68f7bb76d4869b7889b4f2fec.
2024-12-11T20:15:23,888 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0000cbe68f7bb76d4869b7889b4f2fec:
Waiting for close lock at 1733948123874Running coprocessor pre-close hooks at 1733948123874Disabling compacts and flushes for region at 1733948123874Disabling writes for close at 1733948123874Writing region close event to WAL at 1733948123884 (+10 ms)Running coprocessor post-close hooks at 1733948123888 (+4 ms)Closed at 1733948123888
2024-12-11T20:15:23,888 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,row0062,1733948101484.0000cbe68f7bb76d4869b7889b4f2fec.
2024-12-11T20:15:23,888 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 607a8af8bf8fceec868c0a5c1d5fefa1, disabling compactions & flushes
2024-12-11T20:15:23,888 INFO  [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1733948101484.607a8af8bf8fceec868c0a5c1d5fefa1.
2024-12-11T20:15:23,888 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1733948101484.607a8af8bf8fceec868c0a5c1d5fefa1.
2024-12-11T20:15:23,888 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1733948101484.607a8af8bf8fceec868c0a5c1d5fefa1. after waiting 0 ms
2024-12-11T20:15:23,888 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1733948101484.607a8af8bf8fceec868c0a5c1d5fefa1.
2024-12-11T20:15:23,888 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733948101484.607a8af8bf8fceec868c0a5c1d5fefa1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/607a8af8bf8fceec868c0a5c1d5fefa1/info/588b5cca3d0c4abba65d01b91f92cc1b.07f5c44b5ba0e29fbf0ec02f39733158->hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/07f5c44b5ba0e29fbf0ec02f39733158/info/588b5cca3d0c4abba65d01b91f92cc1b-bottom] to archive
2024-12-11T20:15:23,889 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733948101484.607a8af8bf8fceec868c0a5c1d5fefa1.-1 {}] backup.HFileArchiver(363): Archiving compacted files.
2024-12-11T20:15:23,891 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/607a8af8bf8fceec868c0a5c1d5fefa1/info/588b5cca3d0c4abba65d01b91f92cc1b.07f5c44b5ba0e29fbf0ec02f39733158 to hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/archive/data/default/TestLogRolling-testLogRolling/607a8af8bf8fceec868c0a5c1d5fefa1/info/588b5cca3d0c4abba65d01b91f92cc1b.07f5c44b5ba0e29fbf0ec02f39733158
2024-12-11T20:15:23,891 WARN  [StoreCloser-TestLogRolling-testLogRolling,,1733948101484.607a8af8bf8fceec868c0a5c1d5fefa1.-1 {}] regionserver.HStore(2414): Failed to report archival of files: []
2024-12-11T20:15:23,894 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/data/default/TestLogRolling-testLogRolling/607a8af8bf8fceec868c0a5c1d5fefa1/recovered.edits/131.seqid, newMaxSeqId=131, maxSeqId=126
2024-12-11T20:15:23,894 INFO  [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1733948101484.607a8af8bf8fceec868c0a5c1d5fefa1.
2024-12-11T20:15:23,894 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 607a8af8bf8fceec868c0a5c1d5fefa1:
Waiting for close lock at 1733948123888Running coprocessor pre-close hooks at 1733948123888Disabling compacts and flushes for region at 1733948123888Disabling writes for close at 1733948123888Writing region close event to WAL at 1733948123891 (+3 ms)Running coprocessor post-close hooks at 1733948123894 (+3 ms)Closed at 1733948123894
2024-12-11T20:15:23,894 DEBUG [RS_CLOSE_REGION-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,,1733948101484.607a8af8bf8fceec868c0a5c1d5fefa1.
2024-12-11T20:15:24,075 INFO  [RS:0;9fe0640122ec:32831 {}] regionserver.HRegionServer(976): stopping server 9fe0640122ec,32831,1733948076323; all regions closed.
2024-12-11T20:15:24,075 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:15:24,075 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:15:24,075 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:15:24,075 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:15:24,076 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:15:24,077 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35725 is added to blk_1073741834_1010 (size=8107)
2024-12-11T20:15:24,078 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741834_1010 (size=8107)
2024-12-11T20:15:24,080 DEBUG [RS:0;9fe0640122ec:32831 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/oldWALs
2024-12-11T20:15:24,080 INFO  [RS:0;9fe0640122ec:32831 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 9fe0640122ec%2C32831%2C1733948076323.meta:.meta(num 1733948077065)
2024-12-11T20:15:24,080 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:15:24,080 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:15:24,080 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:15:24,080 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:15:24,080 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:15:24,082 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35725 is added to blk_1073741877_1053 (size=780)
2024-12-11T20:15:24,082 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741877_1053 (size=780)
2024-12-11T20:15:24,084 DEBUG [RS:0;9fe0640122ec:32831 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/oldWALs
2024-12-11T20:15:24,084 INFO  [RS:0;9fe0640122ec:32831 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 9fe0640122ec%2C32831%2C1733948076323:(num 1733948123861)
2024-12-11T20:15:24,084 DEBUG [RS:0;9fe0640122ec:32831 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T20:15:24,084 INFO  [RS:0;9fe0640122ec:32831 {}] regionserver.LeaseManager(133): Closed leases
2024-12-11T20:15:24,084 INFO  [RS:0;9fe0640122ec:32831 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-11T20:15:24,084 INFO  [RS:0;9fe0640122ec:32831 {}] hbase.ChoreService(370): Chore service for: regionserver/9fe0640122ec:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown
2024-12-11T20:15:24,084 INFO  [RS:0;9fe0640122ec:32831 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-11T20:15:24,084 INFO  [regionserver/9fe0640122ec:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-11T20:15:24,084 INFO  [RS:0;9fe0640122ec:32831 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:32831
2024-12-11T20:15:24,086 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32831-0x100cafa036a0001, quorum=127.0.0.1:53734, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9fe0640122ec,32831,1733948076323
2024-12-11T20:15:24,086 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46569-0x100cafa036a0000, quorum=127.0.0.1:53734, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-11T20:15:24,086 INFO  [RS:0;9fe0640122ec:32831 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-11T20:15:24,088 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9fe0640122ec,32831,1733948076323]
2024-12-11T20:15:24,089 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9fe0640122ec,32831,1733948076323 already deleted, retry=false
2024-12-11T20:15:24,089 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 9fe0640122ec,32831,1733948076323 expired; onlineServers=0
2024-12-11T20:15:24,089 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '9fe0640122ec,46569,1733948076281' *****
2024-12-11T20:15:24,089 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-11T20:15:24,089 INFO  [M:0;9fe0640122ec:46569 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-11T20:15:24,089 INFO  [M:0;9fe0640122ec:46569 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-11T20:15:24,090 DEBUG [M:0;9fe0640122ec:46569 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-11T20:15:24,090 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-11T20:15:24,090 DEBUG [M:0;9fe0640122ec:46569 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-11T20:15:24,090 DEBUG [master/9fe0640122ec:0:becomeActiveMaster-HFileCleaner.large.0-1733948076467 {}] cleaner.HFileCleaner(306): Exit Thread[master/9fe0640122ec:0:becomeActiveMaster-HFileCleaner.large.0-1733948076467,5,FailOnTimeoutGroup]
2024-12-11T20:15:24,090 DEBUG [master/9fe0640122ec:0:becomeActiveMaster-HFileCleaner.small.0-1733948076467 {}] cleaner.HFileCleaner(306): Exit Thread[master/9fe0640122ec:0:becomeActiveMaster-HFileCleaner.small.0-1733948076467,5,FailOnTimeoutGroup]
2024-12-11T20:15:24,090 INFO  [M:0;9fe0640122ec:46569 {}] hbase.ChoreService(370): Chore service for: master/9fe0640122ec:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-12-11T20:15:24,090 INFO  [M:0;9fe0640122ec:46569 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-11T20:15:24,090 DEBUG [M:0;9fe0640122ec:46569 {}] master.HMaster(1795): Stopping service threads
2024-12-11T20:15:24,090 INFO  [M:0;9fe0640122ec:46569 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-11T20:15:24,090 INFO  [M:0;9fe0640122ec:46569 {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-11T20:15:24,090 ERROR [M:0;9fe0640122ec:46569 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT
java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10]
    Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:37589,5,PEWorkerGroup]
2024-12-11T20:15:24,090 INFO  [M:0;9fe0640122ec:46569 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-11T20:15:24,091 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-11T20:15:24,091 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46569-0x100cafa036a0000, quorum=127.0.0.1:53734, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-11T20:15:24,091 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46569-0x100cafa036a0000, quorum=127.0.0.1:53734, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:15:24,091 DEBUG [M:0;9fe0640122ec:46569 {}] zookeeper.ZKUtil(347): master:46569-0x100cafa036a0000, quorum=127.0.0.1:53734, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-11T20:15:24,091 WARN  [M:0;9fe0640122ec:46569 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-11T20:15:24,092 INFO  [M:0;9fe0640122ec:46569 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/.lastflushedseqids
2024-12-11T20:15:24,102 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35725 is added to blk_1073741878_1054 (size=228)
2024-12-11T20:15:24,102 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741878_1054 (size=228)
2024-12-11T20:15:24,103 INFO  [M:0;9fe0640122ec:46569 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-12-11T20:15:24,103 INFO  [M:0;9fe0640122ec:46569 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-11T20:15:24,103 DEBUG [M:0;9fe0640122ec:46569 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-11T20:15:24,103 INFO  [M:0;9fe0640122ec:46569 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T20:15:24,103 DEBUG [M:0;9fe0640122ec:46569 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T20:15:24,103 DEBUG [M:0;9fe0640122ec:46569 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-11T20:15:24,103 DEBUG [M:0;9fe0640122ec:46569 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T20:15:24,103 INFO  [M:0;9fe0640122ec:46569 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=53.71 KB heapSize=65.93 KB
2024-12-11T20:15:24,120 DEBUG [M:0;9fe0640122ec:46569 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f2ba422a5c6a4e44aac8b293cc69b506 is 82, key is hbase:meta,,1/info:regioninfo/1733948077090/Put/seqid=0
2024-12-11T20:15:24,124 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35725 is added to blk_1073741879_1055 (size=5672)
2024-12-11T20:15:24,124 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741879_1055 (size=5672)
2024-12-11T20:15:24,124 INFO  [M:0;9fe0640122ec:46569 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f2ba422a5c6a4e44aac8b293cc69b506
2024-12-11T20:15:24,144 DEBUG [M:0;9fe0640122ec:46569 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e7a1f3d7a89341cabd91ed071552cf3b is 751, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733948077531/Put/seqid=0
2024-12-11T20:15:24,148 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35725 is added to blk_1073741880_1056 (size=7681)
2024-12-11T20:15:24,148 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741880_1056 (size=7681)
2024-12-11T20:15:24,149 INFO  [M:0;9fe0640122ec:46569 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.11 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e7a1f3d7a89341cabd91ed071552cf3b
2024-12-11T20:15:24,153 INFO  [M:0;9fe0640122ec:46569 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e7a1f3d7a89341cabd91ed071552cf3b
2024-12-11T20:15:24,167 DEBUG [M:0;9fe0640122ec:46569 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b56509f3df524b2cb674dc777790b30f is 69, key is 9fe0640122ec,32831,1733948076323/rs:state/1733948076562/Put/seqid=0
2024-12-11T20:15:24,171 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741881_1057 (size=5156)
2024-12-11T20:15:24,172 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35725 is added to blk_1073741881_1057 (size=5156)
2024-12-11T20:15:24,172 INFO  [M:0;9fe0640122ec:46569 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b56509f3df524b2cb674dc777790b30f
2024-12-11T20:15:24,188 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32831-0x100cafa036a0001, quorum=127.0.0.1:53734, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-11T20:15:24,188 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32831-0x100cafa036a0001, quorum=127.0.0.1:53734, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-11T20:15:24,188 INFO  [RS:0;9fe0640122ec:32831 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-11T20:15:24,188 INFO  [RS:0;9fe0640122ec:32831 {}] regionserver.HRegionServer(1031): Exiting; stopping=9fe0640122ec,32831,1733948076323; zookeeper connection closed.
2024-12-11T20:15:24,188 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@b780752 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@b780752
2024-12-11T20:15:24,189 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-12-11T20:15:24,191 DEBUG [M:0;9fe0640122ec:46569 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/cb95741bb5a34405ad2dba699d771b98 is 52, key is load_balancer_on/state:d/1733948077152/Put/seqid=0
2024-12-11T20:15:24,195 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741882_1058 (size=5056)
2024-12-11T20:15:24,196 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35725 is added to blk_1073741882_1058 (size=5056)
2024-12-11T20:15:24,196 INFO  [M:0;9fe0640122ec:46569 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/cb95741bb5a34405ad2dba699d771b98
2024-12-11T20:15:24,201 DEBUG [M:0;9fe0640122ec:46569 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f2ba422a5c6a4e44aac8b293cc69b506 as hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f2ba422a5c6a4e44aac8b293cc69b506
2024-12-11T20:15:24,206 INFO  [M:0;9fe0640122ec:46569 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f2ba422a5c6a4e44aac8b293cc69b506, entries=8, sequenceid=129, filesize=5.5 K
2024-12-11T20:15:24,207 DEBUG [M:0;9fe0640122ec:46569 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e7a1f3d7a89341cabd91ed071552cf3b as hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e7a1f3d7a89341cabd91ed071552cf3b
2024-12-11T20:15:24,211 INFO  [M:0;9fe0640122ec:46569 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e7a1f3d7a89341cabd91ed071552cf3b
2024-12-11T20:15:24,211 INFO  [M:0;9fe0640122ec:46569 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e7a1f3d7a89341cabd91ed071552cf3b, entries=14, sequenceid=129, filesize=7.5 K
2024-12-11T20:15:24,212 DEBUG [M:0;9fe0640122ec:46569 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b56509f3df524b2cb674dc777790b30f as hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b56509f3df524b2cb674dc777790b30f
2024-12-11T20:15:24,215 INFO  [M:0;9fe0640122ec:46569 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b56509f3df524b2cb674dc777790b30f, entries=1, sequenceid=129, filesize=5.0 K
2024-12-11T20:15:24,216 DEBUG [M:0;9fe0640122ec:46569 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/cb95741bb5a34405ad2dba699d771b98 as hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/cb95741bb5a34405ad2dba699d771b98
2024-12-11T20:15:24,220 INFO  [M:0;9fe0640122ec:46569 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37589/user/jenkins/test-data/eef5dfba-9857-83ca-963a-35659bd7e39c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/cb95741bb5a34405ad2dba699d771b98, entries=1, sequenceid=129, filesize=4.9 K
2024-12-11T20:15:24,221 INFO  [M:0;9fe0640122ec:46569 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.71 KB/54997, heapSize ~65.87 KB/67448, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 118ms, sequenceid=129, compaction requested=false
2024-12-11T20:15:24,223 INFO  [M:0;9fe0640122ec:46569 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T20:15:24,223 DEBUG [M:0;9fe0640122ec:46569 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733948124103Disabling compacts and flushes for region at 1733948124103Disabling writes for close at 1733948124103Obtaining lock to block concurrent updates at 1733948124103Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733948124103Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=54997, getHeapSize=67448, getOffHeapSize=0, getCellsCount=152 at 1733948124104 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733948124104Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733948124104Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733948124119 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733948124119Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733948124129 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733948124143 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733948124143Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733948124153 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733948124167 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733948124167Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733948124176 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733948124191 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733948124191Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@13bfb4f1: reopening flushed file at 1733948124200 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@24936b1f: reopening flushed file at 1733948124206 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@11b0f57: reopening flushed file at 1733948124211 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4e1d789c: reopening flushed file at 1733948124216 (+5 ms)Finished flush of dataSize ~53.71 KB/54997, heapSize ~65.87 KB/67448, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 118ms, sequenceid=129, compaction requested=false at 1733948124221 (+5 ms)Writing region close event to WAL at 1733948124223 (+2 ms)Closed at 1733948124223
2024-12-11T20:15:24,223 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:15:24,223 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:15:24,224 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:15:24,224 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:15:24,224 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:15:24,225 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35725 is added to blk_1073741830_1006 (size=63927)
2024-12-11T20:15:24,226 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33119 is added to blk_1073741830_1006 (size=63927)
2024-12-11T20:15:24,226 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-11T20:15:24,226 INFO  [M:0;9fe0640122ec:46569 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-12-11T20:15:24,226 INFO  [M:0;9fe0640122ec:46569 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46569
2024-12-11T20:15:24,227 INFO  [M:0;9fe0640122ec:46569 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-11T20:15:24,329 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46569-0x100cafa036a0000, quorum=127.0.0.1:53734, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-11T20:15:24,329 INFO  [M:0;9fe0640122ec:46569 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-11T20:15:24,329 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46569-0x100cafa036a0000, quorum=127.0.0.1:53734, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-11T20:15:24,331 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@601b78f7{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T20:15:24,331 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@35a5806e{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-11T20:15:24,332 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-11T20:15:24,332 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7cd1acca{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-11T20:15:24,332 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@18d1ee92{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90893ce-dd95-6d20-b72c-9d6839befe15/hadoop.log.dir/,STOPPED}
2024-12-11T20:15:24,333 WARN  [BP-1861006598-172.17.0.2-1733948075590 heartbeating to localhost/127.0.0.1:37589 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-11T20:15:24,333 WARN  [BP-1861006598-172.17.0.2-1733948075590 heartbeating to localhost/127.0.0.1:37589 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1861006598-172.17.0.2-1733948075590 (Datanode Uuid 21266db9-4349-4f71-97ad-93cb37c31e02) service to localhost/127.0.0.1:37589
2024-12-11T20:15:24,334 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-11T20:15:24,334 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-11T20:15:24,334 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90893ce-dd95-6d20-b72c-9d6839befe15/cluster_7587c867-25df-d69c-d06b-fd5497de92d4/data/data3/current/BP-1861006598-172.17.0.2-1733948075590 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T20:15:24,334 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90893ce-dd95-6d20-b72c-9d6839befe15/cluster_7587c867-25df-d69c-d06b-fd5497de92d4/data/data4/current/BP-1861006598-172.17.0.2-1733948075590 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T20:15:24,334 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-11T20:15:24,337 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@76385d85{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T20:15:24,338 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@284a8a01{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-11T20:15:24,338 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-11T20:15:24,338 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7193a060{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-11T20:15:24,338 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@444d0b71{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90893ce-dd95-6d20-b72c-9d6839befe15/hadoop.log.dir/,STOPPED}
2024-12-11T20:15:24,340 WARN  [BP-1861006598-172.17.0.2-1733948075590 heartbeating to localhost/127.0.0.1:37589 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-11T20:15:24,340 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-11T20:15:24,340 WARN  [BP-1861006598-172.17.0.2-1733948075590 heartbeating to localhost/127.0.0.1:37589 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1861006598-172.17.0.2-1733948075590 (Datanode Uuid 5bcb4a8d-8e7a-44b6-8f9a-f9e4d482fcf2) service to localhost/127.0.0.1:37589
2024-12-11T20:15:24,340 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-11T20:15:24,340 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90893ce-dd95-6d20-b72c-9d6839befe15/cluster_7587c867-25df-d69c-d06b-fd5497de92d4/data/data1/current/BP-1861006598-172.17.0.2-1733948075590 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T20:15:24,340 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90893ce-dd95-6d20-b72c-9d6839befe15/cluster_7587c867-25df-d69c-d06b-fd5497de92d4/data/data2/current/BP-1861006598-172.17.0.2-1733948075590 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T20:15:24,341 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-11T20:15:24,346 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@10e59f6e{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-11T20:15:24,347 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@56d12e32{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-11T20:15:24,347 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-11T20:15:24,347 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@40a5a9d2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-11T20:15:24,347 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2f05aefd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90893ce-dd95-6d20-b72c-9d6839befe15/hadoop.log.dir/,STOPPED}
2024-12-11T20:15:24,351 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:15:24,351 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:15:24,354 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-12-11T20:15:24,385 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-12-11T20:15:24,397 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRolling Thread=239 (was 208)
Potentially hanging thread: nioEventLoopGroup-40-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-39-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-41-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-20
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37589
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-39-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-41-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-21
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-14-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:37589 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.6@localhost:37589
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-38-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37589
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:37589 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37589
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-41-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-38-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-17
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-18
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:37589
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-38-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:37589 from jenkins.hfs.6
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-40-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-16
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-40-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-14-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-39-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37589
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-15
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-14-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-19
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=515 (was 483) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=71 (was 72), ProcessCount=11 (was 11), AvailableMemoryMB=280 (was 316)
2024-12-11T20:15:24,406 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=239, OpenFileDescriptor=515, MaxFileDescriptor=1048576, SystemLoadAverage=71, ProcessCount=11, AvailableMemoryMB=280
2024-12-11T20:15:24,406 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-11T20:15:24,406 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90893ce-dd95-6d20-b72c-9d6839befe15/hadoop.log.dir so I do NOT create it in target/test-data/2b1c9106-caf5-ed09-2d13-05be0c65ac5f
2024-12-11T20:15:24,406 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c90893ce-dd95-6d20-b72c-9d6839befe15/hadoop.tmp.dir so I do NOT create it in target/test-data/2b1c9106-caf5-ed09-2d13-05be0c65ac5f
2024-12-11T20:15:24,406 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2b1c9106-caf5-ed09-2d13-05be0c65ac5f/cluster_f31c84cd-b048-50a9-4468-390e6aa69c32, deleteOnExit=true
2024-12-11T20:15:24,406 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-12-11T20:15:24,407 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2b1c9106-caf5-ed09-2d13-05be0c65ac5f/test.cache.data in system properties and HBase conf
2024-12-11T20:15:24,407 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2b1c9106-caf5-ed09-2d13-05be0c65ac5f/hadoop.tmp.dir in system properties and HBase conf
2024-12-11T20:15:24,407 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2b1c9106-caf5-ed09-2d13-05be0c65ac5f/hadoop.log.dir in system properties and HBase conf
2024-12-11T20:15:24,407 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2b1c9106-caf5-ed09-2d13-05be0c65ac5f/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-11T20:15:24,407 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2b1c9106-caf5-ed09-2d13-05be0c65ac5f/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-11T20:15:24,407 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-12-11T20:15:24,407 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-11T20:15:24,407 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2b1c9106-caf5-ed09-2d13-05be0c65ac5f/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-11T20:15:24,407 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2b1c9106-caf5-ed09-2d13-05be0c65ac5f/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-11T20:15:24,407 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2b1c9106-caf5-ed09-2d13-05be0c65ac5f/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-11T20:15:24,407 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2b1c9106-caf5-ed09-2d13-05be0c65ac5f/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-11T20:15:24,407 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2b1c9106-caf5-ed09-2d13-05be0c65ac5f/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-11T20:15:24,408 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2b1c9106-caf5-ed09-2d13-05be0c65ac5f/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-11T20:15:24,408 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2b1c9106-caf5-ed09-2d13-05be0c65ac5f/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-11T20:15:24,408 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2b1c9106-caf5-ed09-2d13-05be0c65ac5f/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-11T20:15:24,408 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2b1c9106-caf5-ed09-2d13-05be0c65ac5f/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-11T20:15:24,408 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2b1c9106-caf5-ed09-2d13-05be0c65ac5f/nfs.dump.dir in system properties and HBase conf
2024-12-11T20:15:24,408 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2b1c9106-caf5-ed09-2d13-05be0c65ac5f/java.io.tmpdir in system properties and HBase conf
2024-12-11T20:15:24,408 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2b1c9106-caf5-ed09-2d13-05be0c65ac5f/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-11T20:15:24,408 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2b1c9106-caf5-ed09-2d13-05be0c65ac5f/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-11T20:15:24,408 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2b1c9106-caf5-ed09-2d13-05be0c65ac5f/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-11T20:15:24,420 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-11T20:15:24,474 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T20:15:24,478 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-11T20:15:24,479 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-11T20:15:24,479 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-11T20:15:24,479 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-11T20:15:24,480 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T20:15:24,480 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2d48d0f6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2b1c9106-caf5-ed09-2d13-05be0c65ac5f/hadoop.log.dir/,AVAILABLE}
2024-12-11T20:15:24,480 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@246d323a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-11T20:15:24,576 INFO  [regionserver/9fe0640122ec:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-11T20:15:24,596 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@49428dc3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2b1c9106-caf5-ed09-2d13-05be0c65ac5f/java.io.tmpdir/jetty-localhost-46619-hadoop-hdfs-3_4_1-tests_jar-_-any-15506388922552457804/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-11T20:15:24,596 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@186dc28{HTTP/1.1, (http/1.1)}{localhost:46619}
2024-12-11T20:15:24,596 INFO  [Time-limited test {}] server.Server(415): Started @288192ms
2024-12-11T20:15:24,610 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-11T20:15:24,656 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T20:15:24,659 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-11T20:15:24,659 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-11T20:15:24,659 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-11T20:15:24,659 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-11T20:15:24,660 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@536a5c19{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2b1c9106-caf5-ed09-2d13-05be0c65ac5f/hadoop.log.dir/,AVAILABLE}
2024-12-11T20:15:24,660 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@352a3917{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-11T20:15:24,774 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@45e68726{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2b1c9106-caf5-ed09-2d13-05be0c65ac5f/java.io.tmpdir/jetty-localhost-45675-hadoop-hdfs-3_4_1-tests_jar-_-any-2731902038460912257/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T20:15:24,775 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6e1cda33{HTTP/1.1, (http/1.1)}{localhost:45675}
2024-12-11T20:15:24,775 INFO  [Time-limited test {}] server.Server(415): Started @288370ms
2024-12-11T20:15:24,776 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-11T20:15:24,805 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-11T20:15:24,807 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-11T20:15:24,808 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-11T20:15:24,808 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-11T20:15:24,808 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-11T20:15:24,808 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@115f614b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2b1c9106-caf5-ed09-2d13-05be0c65ac5f/hadoop.log.dir/,AVAILABLE}
2024-12-11T20:15:24,809 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@392cf3b8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-11T20:15:24,855 WARN  [Thread-2474 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2b1c9106-caf5-ed09-2d13-05be0c65ac5f/cluster_f31c84cd-b048-50a9-4468-390e6aa69c32/data/data1/current/BP-1466799083-172.17.0.2-1733948124427/current, will proceed with Du for space computation calculation, 
2024-12-11T20:15:24,855 WARN  [Thread-2475 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2b1c9106-caf5-ed09-2d13-05be0c65ac5f/cluster_f31c84cd-b048-50a9-4468-390e6aa69c32/data/data2/current/BP-1466799083-172.17.0.2-1733948124427/current, will proceed with Du for space computation calculation, 
2024-12-11T20:15:24,876 WARN  [Thread-2453 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-11T20:15:24,878 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3288592c88773614 with lease ID 0x458b519c6c8583f2: Processing first storage report for DS-ab622763-fea9-41c0-8085-e6fb00fe7140 from datanode DatanodeRegistration(127.0.0.1:43549, datanodeUuid=c5f0568f-755c-49bf-9570-11811c1e5b15, infoPort=34263, infoSecurePort=0, ipcPort=44157, storageInfo=lv=-57;cid=testClusterID;nsid=2021192783;c=1733948124427)
2024-12-11T20:15:24,878 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3288592c88773614 with lease ID 0x458b519c6c8583f2: from storage DS-ab622763-fea9-41c0-8085-e6fb00fe7140 node DatanodeRegistration(127.0.0.1:43549, datanodeUuid=c5f0568f-755c-49bf-9570-11811c1e5b15, infoPort=34263, infoSecurePort=0, ipcPort=44157, storageInfo=lv=-57;cid=testClusterID;nsid=2021192783;c=1733948124427), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-11T20:15:24,878 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3288592c88773614 with lease ID 0x458b519c6c8583f2: Processing first storage report for DS-66444471-eb9f-438f-b612-a818cc16446c from datanode DatanodeRegistration(127.0.0.1:43549, datanodeUuid=c5f0568f-755c-49bf-9570-11811c1e5b15, infoPort=34263, infoSecurePort=0, ipcPort=44157, storageInfo=lv=-57;cid=testClusterID;nsid=2021192783;c=1733948124427)
2024-12-11T20:15:24,878 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3288592c88773614 with lease ID 0x458b519c6c8583f2: from storage DS-66444471-eb9f-438f-b612-a818cc16446c node DatanodeRegistration(127.0.0.1:43549, datanodeUuid=c5f0568f-755c-49bf-9570-11811c1e5b15, infoPort=34263, infoSecurePort=0, ipcPort=44157, storageInfo=lv=-57;cid=testClusterID;nsid=2021192783;c=1733948124427), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T20:15:24,926 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2d8770d2{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2b1c9106-caf5-ed09-2d13-05be0c65ac5f/java.io.tmpdir/jetty-localhost-38717-hadoop-hdfs-3_4_1-tests_jar-_-any-1125174036820456082/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T20:15:24,926 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@78791d2c{HTTP/1.1, (http/1.1)}{localhost:38717}
2024-12-11T20:15:24,926 INFO  [Time-limited test {}] server.Server(415): Started @288522ms
2024-12-11T20:15:24,927 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-11T20:15:25,004 WARN  [Thread-2500 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2b1c9106-caf5-ed09-2d13-05be0c65ac5f/cluster_f31c84cd-b048-50a9-4468-390e6aa69c32/data/data3/current/BP-1466799083-172.17.0.2-1733948124427/current, will proceed with Du for space computation calculation, 
2024-12-11T20:15:25,004 WARN  [Thread-2501 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2b1c9106-caf5-ed09-2d13-05be0c65ac5f/cluster_f31c84cd-b048-50a9-4468-390e6aa69c32/data/data4/current/BP-1466799083-172.17.0.2-1733948124427/current, will proceed with Du for space computation calculation, 
2024-12-11T20:15:25,028 WARN  [Thread-2489 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-11T20:15:25,030 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9496d01ef1933ac7 with lease ID 0x458b519c6c8583f3: Processing first storage report for DS-701e9de8-5415-45a0-bdc7-33ba6b14d126 from datanode DatanodeRegistration(127.0.0.1:33713, datanodeUuid=dd1e72ec-36be-4275-876e-e3d1a433898c, infoPort=45645, infoSecurePort=0, ipcPort=34349, storageInfo=lv=-57;cid=testClusterID;nsid=2021192783;c=1733948124427)
2024-12-11T20:15:25,030 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9496d01ef1933ac7 with lease ID 0x458b519c6c8583f3: from storage DS-701e9de8-5415-45a0-bdc7-33ba6b14d126 node DatanodeRegistration(127.0.0.1:33713, datanodeUuid=dd1e72ec-36be-4275-876e-e3d1a433898c, infoPort=45645, infoSecurePort=0, ipcPort=34349, storageInfo=lv=-57;cid=testClusterID;nsid=2021192783;c=1733948124427), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T20:15:25,030 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9496d01ef1933ac7 with lease ID 0x458b519c6c8583f3: Processing first storage report for DS-47f4177e-63e7-4efc-9bca-31b04722c310 from datanode DatanodeRegistration(127.0.0.1:33713, datanodeUuid=dd1e72ec-36be-4275-876e-e3d1a433898c, infoPort=45645, infoSecurePort=0, ipcPort=34349, storageInfo=lv=-57;cid=testClusterID;nsid=2021192783;c=1733948124427)
2024-12-11T20:15:25,030 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9496d01ef1933ac7 with lease ID 0x458b519c6c8583f3: from storage DS-47f4177e-63e7-4efc-9bca-31b04722c310 node DatanodeRegistration(127.0.0.1:33713, datanodeUuid=dd1e72ec-36be-4275-876e-e3d1a433898c, infoPort=45645, infoSecurePort=0, ipcPort=34349, storageInfo=lv=-57;cid=testClusterID;nsid=2021192783;c=1733948124427), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-11T20:15:25,049 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2b1c9106-caf5-ed09-2d13-05be0c65ac5f
2024-12-11T20:15:25,052 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2b1c9106-caf5-ed09-2d13-05be0c65ac5f/cluster_f31c84cd-b048-50a9-4468-390e6aa69c32/zookeeper_0, clientPort=56007, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2b1c9106-caf5-ed09-2d13-05be0c65ac5f/cluster_f31c84cd-b048-50a9-4468-390e6aa69c32/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2b1c9106-caf5-ed09-2d13-05be0c65ac5f/cluster_f31c84cd-b048-50a9-4468-390e6aa69c32/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-11T20:15:25,052 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=56007
2024-12-11T20:15:25,053 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T20:15:25,054 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T20:15:25,062 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33713 is added to blk_1073741825_1001 (size=7)
2024-12-11T20:15:25,062 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43549 is added to blk_1073741825_1001 (size=7)
2024-12-11T20:15:25,063 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:37155/user/jenkins/test-data/beeabada-bbfd-e2a4-53bf-1c2a18b752dc with version=8
2024-12-11T20:15:25,063 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:36219/user/jenkins/test-data/f5b2139b-d9bf-1ac8-e21d-25c62b368738/hbase-staging
2024-12-11T20:15:25,065 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/9fe0640122ec:0 server-side Connection retries=45
2024-12-11T20:15:25,065 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-11T20:15:25,065 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-11T20:15:25,065 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-11T20:15:25,065 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-11T20:15:25,065 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-11T20:15:25,065 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-12-11T20:15:25,066 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-11T20:15:25,066 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36929
2024-12-11T20:15:25,067 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36929 connecting to ZooKeeper ensemble=127.0.0.1:56007
2024-12-11T20:15:25,072 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:369290x0, quorum=127.0.0.1:56007, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-11T20:15:25,073 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36929-0x100cafac1fa0000 connected
2024-12-11T20:15:25,089 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T20:15:25,090 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T20:15:25,093 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36929-0x100cafac1fa0000, quorum=127.0.0.1:56007, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-11T20:15:25,093 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:37155/user/jenkins/test-data/beeabada-bbfd-e2a4-53bf-1c2a18b752dc, hbase.cluster.distributed=false
2024-12-11T20:15:25,095 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36929-0x100cafac1fa0000, quorum=127.0.0.1:56007, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-11T20:15:25,095 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36929
2024-12-11T20:15:25,095 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36929
2024-12-11T20:15:25,096 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36929
2024-12-11T20:15:25,096 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36929
2024-12-11T20:15:25,096 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36929
2024-12-11T20:15:25,112 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/9fe0640122ec:0 server-side Connection retries=45
2024-12-11T20:15:25,112 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-11T20:15:25,112 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-11T20:15:25,112 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-11T20:15:25,112 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-11T20:15:25,112 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-11T20:15:25,112 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-11T20:15:25,112 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-11T20:15:25,113 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41281
2024-12-11T20:15:25,114 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41281 connecting to ZooKeeper ensemble=127.0.0.1:56007
2024-12-11T20:15:25,114 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T20:15:25,116 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T20:15:25,120 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:412810x0, quorum=127.0.0.1:56007, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-11T20:15:25,120 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:412810x0, quorum=127.0.0.1:56007, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-11T20:15:25,120 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41281-0x100cafac1fa0001 connected
2024-12-11T20:15:25,120 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-11T20:15:25,122 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-11T20:15:25,122 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41281-0x100cafac1fa0001, quorum=127.0.0.1:56007, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-11T20:15:25,123 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41281-0x100cafac1fa0001, quorum=127.0.0.1:56007, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-11T20:15:25,124 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41281
2024-12-11T20:15:25,125 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41281
2024-12-11T20:15:25,126 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41281
2024-12-11T20:15:25,128 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41281
2024-12-11T20:15:25,128 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41281
2024-12-11T20:15:25,140 DEBUG [M:0;9fe0640122ec:36929 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;9fe0640122ec:36929
2024-12-11T20:15:25,140 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/9fe0640122ec,36929,1733948125065
2024-12-11T20:15:25,142 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36929-0x100cafac1fa0000, quorum=127.0.0.1:56007, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-11T20:15:25,142 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41281-0x100cafac1fa0001, quorum=127.0.0.1:56007, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-11T20:15:25,142 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36929-0x100cafac1fa0000, quorum=127.0.0.1:56007, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/9fe0640122ec,36929,1733948125065
2024-12-11T20:15:25,144 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41281-0x100cafac1fa0001, quorum=127.0.0.1:56007, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-11T20:15:25,144 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36929-0x100cafac1fa0000, quorum=127.0.0.1:56007, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:15:25,144 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41281-0x100cafac1fa0001, quorum=127.0.0.1:56007, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:15:25,144 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36929-0x100cafac1fa0000, quorum=127.0.0.1:56007, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-11T20:15:25,145 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/9fe0640122ec,36929,1733948125065 from backup master directory
2024-12-11T20:15:25,146 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36929-0x100cafac1fa0000, quorum=127.0.0.1:56007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/9fe0640122ec,36929,1733948125065
2024-12-11T20:15:25,146 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41281-0x100cafac1fa0001, quorum=127.0.0.1:56007, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-11T20:15:25,146 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36929-0x100cafac1fa0000, quorum=127.0.0.1:56007, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-11T20:15:25,146 WARN  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-11T20:15:25,146 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=9fe0640122ec,36929,1733948125065
2024-12-11T20:15:25,152 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:37155/user/jenkins/test-data/beeabada-bbfd-e2a4-53bf-1c2a18b752dc/hbase.id] with ID: 91817917-8452-4028-9583-c6636f95f57b
2024-12-11T20:15:25,152 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:37155/user/jenkins/test-data/beeabada-bbfd-e2a4-53bf-1c2a18b752dc/.tmp/hbase.id
2024-12-11T20:15:25,158 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33713 is added to blk_1073741826_1002 (size=42)
2024-12-11T20:15:25,158 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43549 is added to blk_1073741826_1002 (size=42)
2024-12-11T20:15:25,159 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:37155/user/jenkins/test-data/beeabada-bbfd-e2a4-53bf-1c2a18b752dc/.tmp/hbase.id]:[hdfs://localhost:37155/user/jenkins/test-data/beeabada-bbfd-e2a4-53bf-1c2a18b752dc/hbase.id]
2024-12-11T20:15:25,168 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T20:15:25,168 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-12-11T20:15:25,169 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-12-11T20:15:25,171 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36929-0x100cafac1fa0000, quorum=127.0.0.1:56007, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:15:25,171 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41281-0x100cafac1fa0001, quorum=127.0.0.1:56007, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:15:25,176 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43549 is added to blk_1073741827_1003 (size=196)
2024-12-11T20:15:25,176 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33713 is added to blk_1073741827_1003 (size=196)
2024-12-11T20:15:25,177 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-11T20:15:25,178 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-11T20:15:25,178 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-11T20:15:25,185 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43549 is added to blk_1073741828_1004 (size=1189)
2024-12-11T20:15:25,185 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33713 is added to blk_1073741828_1004 (size=1189)
2024-12-11T20:15:25,351 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:15:25,351 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:15:25,586 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:37155/user/jenkins/test-data/beeabada-bbfd-e2a4-53bf-1c2a18b752dc/MasterData/data/master/store
2024-12-11T20:15:25,592 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43549 is added to blk_1073741829_1005 (size=34)
2024-12-11T20:15:25,592 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33713 is added to blk_1073741829_1005 (size=34)
2024-12-11T20:15:25,593 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T20:15:25,593 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-11T20:15:25,593 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T20:15:25,593 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T20:15:25,593 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-11T20:15:25,593 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T20:15:25,593 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T20:15:25,593 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733948125593Disabling compacts and flushes for region at 1733948125593Disabling writes for close at 1733948125593Writing region close event to WAL at 1733948125593Closed at 1733948125593
2024-12-11T20:15:25,594 WARN  [master/9fe0640122ec:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:37155/user/jenkins/test-data/beeabada-bbfd-e2a4-53bf-1c2a18b752dc/MasterData/data/master/store/.initializing
2024-12-11T20:15:25,594 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:37155/user/jenkins/test-data/beeabada-bbfd-e2a4-53bf-1c2a18b752dc/MasterData/WALs/9fe0640122ec,36929,1733948125065
2024-12-11T20:15:25,596 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9fe0640122ec%2C36929%2C1733948125065, suffix=, logDir=hdfs://localhost:37155/user/jenkins/test-data/beeabada-bbfd-e2a4-53bf-1c2a18b752dc/MasterData/WALs/9fe0640122ec,36929,1733948125065, archiveDir=hdfs://localhost:37155/user/jenkins/test-data/beeabada-bbfd-e2a4-53bf-1c2a18b752dc/MasterData/oldWALs, maxLogs=10
2024-12-11T20:15:25,597 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9fe0640122ec%2C36929%2C1733948125065.1733948125596
2024-12-11T20:15:25,601 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/beeabada-bbfd-e2a4-53bf-1c2a18b752dc/MasterData/WALs/9fe0640122ec,36929,1733948125065/9fe0640122ec%2C36929%2C1733948125065.1733948125596
2024-12-11T20:15:25,605 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34263:34263),(127.0.0.1/127.0.0.1:45645:45645)]
2024-12-11T20:15:25,608 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-11T20:15:25,609 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T20:15:25,609 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T20:15:25,609 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T20:15:25,610 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-11T20:15:25,611 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-11T20:15:25,611 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:15:25,612 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T20:15:25,612 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-11T20:15:25,613 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-11T20:15:25,613 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:15:25,614 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-11T20:15:25,614 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-11T20:15:25,615 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-11T20:15:25,615 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:15:25,615 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-11T20:15:25,615 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-11T20:15:25,616 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-11T20:15:25,616 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:15:25,617 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-11T20:15:25,617 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T20:15:25,617 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37155/user/jenkins/test-data/beeabada-bbfd-e2a4-53bf-1c2a18b752dc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-11T20:15:25,618 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37155/user/jenkins/test-data/beeabada-bbfd-e2a4-53bf-1c2a18b752dc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-11T20:15:25,619 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T20:15:25,619 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T20:15:25,619 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-11T20:15:25,620 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-11T20:15:25,622 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37155/user/jenkins/test-data/beeabada-bbfd-e2a4-53bf-1c2a18b752dc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-11T20:15:25,622 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=824966, jitterRate=0.04899954795837402}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-11T20:15:25,623 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1733948125609Initializing all the Stores at 1733948125610 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733948125610Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733948125610Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733948125610Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733948125610Cleaning up temporary data from old regions at 1733948125619 (+9 ms)Region opened successfully at 1733948125623 (+4 ms)
2024-12-11T20:15:25,623 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-11T20:15:25,626 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@323940b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9fe0640122ec/172.17.0.2:0
2024-12-11T20:15:25,627 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-12-11T20:15:25,627 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-11T20:15:25,627 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-11T20:15:25,627 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-11T20:15:25,627 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-12-11T20:15:25,628 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-12-11T20:15:25,628 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-11T20:15:25,630 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-11T20:15:25,631 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36929-0x100cafac1fa0000, quorum=127.0.0.1:56007, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-11T20:15:25,632 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-12-11T20:15:25,632 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-11T20:15:25,633 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36929-0x100cafac1fa0000, quorum=127.0.0.1:56007, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-11T20:15:25,634 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-12-11T20:15:25,634 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-11T20:15:25,635 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36929-0x100cafac1fa0000, quorum=127.0.0.1:56007, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-11T20:15:25,636 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-12-11T20:15:25,636 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36929-0x100cafac1fa0000, quorum=127.0.0.1:56007, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-11T20:15:25,637 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-12-11T20:15:25,639 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36929-0x100cafac1fa0000, quorum=127.0.0.1:56007, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-11T20:15:25,640 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-11T20:15:25,641 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36929-0x100cafac1fa0000, quorum=127.0.0.1:56007, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-11T20:15:25,641 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41281-0x100cafac1fa0001, quorum=127.0.0.1:56007, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-11T20:15:25,642 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36929-0x100cafac1fa0000, quorum=127.0.0.1:56007, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:15:25,642 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41281-0x100cafac1fa0001, quorum=127.0.0.1:56007, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:15:25,642 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=9fe0640122ec,36929,1733948125065, sessionid=0x100cafac1fa0000, setting cluster-up flag (Was=false)
2024-12-11T20:15:25,644 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36929-0x100cafac1fa0000, quorum=127.0.0.1:56007, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:15:25,644 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41281-0x100cafac1fa0001, quorum=127.0.0.1:56007, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:15:25,648 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-11T20:15:25,649 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9fe0640122ec,36929,1733948125065
2024-12-11T20:15:25,652 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36929-0x100cafac1fa0000, quorum=127.0.0.1:56007, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:15:25,652 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41281-0x100cafac1fa0001, quorum=127.0.0.1:56007, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:15:25,656 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-11T20:15:25,657 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9fe0640122ec,36929,1733948125065
2024-12-11T20:15:25,658 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:37155/user/jenkins/test-data/beeabada-bbfd-e2a4-53bf-1c2a18b752dc/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-12-11T20:15:25,659 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-12-11T20:15:25,660 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-12-11T20:15:25,660 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-11T20:15:25,660 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 9fe0640122ec,36929,1733948125065
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-11T20:15:25,661 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/9fe0640122ec:0, corePoolSize=5, maxPoolSize=5
2024-12-11T20:15:25,661 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/9fe0640122ec:0, corePoolSize=5, maxPoolSize=5
2024-12-11T20:15:25,661 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/9fe0640122ec:0, corePoolSize=5, maxPoolSize=5
2024-12-11T20:15:25,661 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/9fe0640122ec:0, corePoolSize=5, maxPoolSize=5
2024-12-11T20:15:25,661 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/9fe0640122ec:0, corePoolSize=10, maxPoolSize=10
2024-12-11T20:15:25,661 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:15:25,661 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/9fe0640122ec:0, corePoolSize=2, maxPoolSize=2
2024-12-11T20:15:25,661 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:15:25,663 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733948155663
2024-12-11T20:15:25,663 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-11T20:15:25,663 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-11T20:15:25,664 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-11T20:15:25,664 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-11T20:15:25,664 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-11T20:15:25,664 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-11T20:15:25,664 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-11T20:15:25,664 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-12-11T20:15:25,664 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-11T20:15:25,665 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:15:25,665 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-11T20:15:25,665 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-11T20:15:25,665 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-11T20:15:25,665 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-11T20:15:25,665 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-11T20:15:25,665 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-11T20:15:25,667 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/9fe0640122ec:0:becomeActiveMaster-HFileCleaner.large.0-1733948125665,5,FailOnTimeoutGroup]
2024-12-11T20:15:25,669 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/9fe0640122ec:0:becomeActiveMaster-HFileCleaner.small.0-1733948125667,5,FailOnTimeoutGroup]
2024-12-11T20:15:25,669 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-11T20:15:25,669 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-11T20:15:25,669 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-11T20:15:25,669 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-11T20:15:25,674 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43549 is added to blk_1073741831_1007 (size=1321)
2024-12-11T20:15:25,675 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33713 is added to blk_1073741831_1007 (size=1321)
2024-12-11T20:15:25,676 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:37155/user/jenkins/test-data/beeabada-bbfd-e2a4-53bf-1c2a18b752dc/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-12-11T20:15:25,676 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:37155/user/jenkins/test-data/beeabada-bbfd-e2a4-53bf-1c2a18b752dc
2024-12-11T20:15:25,681 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43549 is added to blk_1073741832_1008 (size=32)
2024-12-11T20:15:25,681 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33713 is added to blk_1073741832_1008 (size=32)
2024-12-11T20:15:25,682 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T20:15:25,683 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-11T20:15:25,684 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-11T20:15:25,684 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:15:25,684 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T20:15:25,685 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-11T20:15:25,685 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-11T20:15:25,685 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:15:25,686 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T20:15:25,686 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-11T20:15:25,687 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-11T20:15:25,687 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:15:25,687 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T20:15:25,687 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-11T20:15:25,688 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-11T20:15:25,688 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:15:25,688 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T20:15:25,689 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-11T20:15:25,689 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37155/user/jenkins/test-data/beeabada-bbfd-e2a4-53bf-1c2a18b752dc/data/hbase/meta/1588230740
2024-12-11T20:15:25,689 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37155/user/jenkins/test-data/beeabada-bbfd-e2a4-53bf-1c2a18b752dc/data/hbase/meta/1588230740
2024-12-11T20:15:25,690 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-11T20:15:25,690 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-11T20:15:25,691 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-11T20:15:25,691 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-11T20:15:25,693 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37155/user/jenkins/test-data/beeabada-bbfd-e2a4-53bf-1c2a18b752dc/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-11T20:15:25,693 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=719416, jitterRate=-0.08521565794944763}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-11T20:15:25,694 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1733948125682Initializing all the Stores at 1733948125683 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733948125683Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733948125683Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733948125683Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733948125683Cleaning up temporary data from old regions at 1733948125690 (+7 ms)Region opened successfully at 1733948125694 (+4 ms)
2024-12-11T20:15:25,694 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-11T20:15:25,694 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-11T20:15:25,694 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-11T20:15:25,694 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-11T20:15:25,694 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-11T20:15:25,694 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-11T20:15:25,694 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733948125694Disabling compacts and flushes for region at 1733948125694Disabling writes for close at 1733948125694Writing region close event to WAL at 1733948125694Closed at 1733948125694
2024-12-11T20:15:25,695 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-11T20:15:25,695 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-12-11T20:15:25,695 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-11T20:15:25,696 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-11T20:15:25,697 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-11T20:15:25,730 INFO  [RS:0;9fe0640122ec:41281 {}] regionserver.HRegionServer(746): ClusterId : 91817917-8452-4028-9583-c6636f95f57b
2024-12-11T20:15:25,730 DEBUG [RS:0;9fe0640122ec:41281 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-11T20:15:25,732 DEBUG [RS:0;9fe0640122ec:41281 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-11T20:15:25,732 DEBUG [RS:0;9fe0640122ec:41281 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-11T20:15:25,733 DEBUG [RS:0;9fe0640122ec:41281 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-11T20:15:25,733 DEBUG [RS:0;9fe0640122ec:41281 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d1fbcc5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9fe0640122ec/172.17.0.2:0
2024-12-11T20:15:25,745 DEBUG [RS:0;9fe0640122ec:41281 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;9fe0640122ec:41281
2024-12-11T20:15:25,745 INFO  [RS:0;9fe0640122ec:41281 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-11T20:15:25,745 INFO  [RS:0;9fe0640122ec:41281 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-11T20:15:25,745 DEBUG [RS:0;9fe0640122ec:41281 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-11T20:15:25,746 INFO  [RS:0;9fe0640122ec:41281 {}] regionserver.HRegionServer(2659): reportForDuty to master=9fe0640122ec,36929,1733948125065 with port=41281, startcode=1733948125111
2024-12-11T20:15:25,746 DEBUG [RS:0;9fe0640122ec:41281 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-11T20:15:25,748 INFO  [HMaster-EventLoopGroup-16-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50525, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.7 (auth:SIMPLE), service=RegionServerStatusService
2024-12-11T20:15:25,749 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36929 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9fe0640122ec,41281,1733948125111
2024-12-11T20:15:25,749 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36929 {}] master.ServerManager(517): Registering regionserver=9fe0640122ec,41281,1733948125111
2024-12-11T20:15:25,750 DEBUG [RS:0;9fe0640122ec:41281 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:37155/user/jenkins/test-data/beeabada-bbfd-e2a4-53bf-1c2a18b752dc
2024-12-11T20:15:25,750 DEBUG [RS:0;9fe0640122ec:41281 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:37155
2024-12-11T20:15:25,750 DEBUG [RS:0;9fe0640122ec:41281 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-11T20:15:25,752 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36929-0x100cafac1fa0000, quorum=127.0.0.1:56007, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-11T20:15:25,752 DEBUG [RS:0;9fe0640122ec:41281 {}] zookeeper.ZKUtil(111): regionserver:41281-0x100cafac1fa0001, quorum=127.0.0.1:56007, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9fe0640122ec,41281,1733948125111
2024-12-11T20:15:25,752 WARN  [RS:0;9fe0640122ec:41281 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-11T20:15:25,752 INFO  [RS:0;9fe0640122ec:41281 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-11T20:15:25,753 DEBUG [RS:0;9fe0640122ec:41281 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:37155/user/jenkins/test-data/beeabada-bbfd-e2a4-53bf-1c2a18b752dc/WALs/9fe0640122ec,41281,1733948125111
2024-12-11T20:15:25,753 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9fe0640122ec,41281,1733948125111]
2024-12-11T20:15:25,755 INFO  [RS:0;9fe0640122ec:41281 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-11T20:15:25,757 INFO  [RS:0;9fe0640122ec:41281 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-11T20:15:25,757 INFO  [RS:0;9fe0640122ec:41281 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-11T20:15:25,757 INFO  [RS:0;9fe0640122ec:41281 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-11T20:15:25,758 INFO  [RS:0;9fe0640122ec:41281 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-11T20:15:25,758 INFO  [RS:0;9fe0640122ec:41281 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-11T20:15:25,758 INFO  [RS:0;9fe0640122ec:41281 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-11T20:15:25,758 DEBUG [RS:0;9fe0640122ec:41281 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:15:25,758 DEBUG [RS:0;9fe0640122ec:41281 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:15:25,759 DEBUG [RS:0;9fe0640122ec:41281 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:15:25,759 DEBUG [RS:0;9fe0640122ec:41281 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:15:25,759 DEBUG [RS:0;9fe0640122ec:41281 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:15:25,759 DEBUG [RS:0;9fe0640122ec:41281 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9fe0640122ec:0, corePoolSize=2, maxPoolSize=2
2024-12-11T20:15:25,759 DEBUG [RS:0;9fe0640122ec:41281 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:15:25,759 DEBUG [RS:0;9fe0640122ec:41281 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:15:25,759 DEBUG [RS:0;9fe0640122ec:41281 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:15:25,759 DEBUG [RS:0;9fe0640122ec:41281 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:15:25,759 DEBUG [RS:0;9fe0640122ec:41281 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:15:25,759 DEBUG [RS:0;9fe0640122ec:41281 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9fe0640122ec:0, corePoolSize=1, maxPoolSize=1
2024-12-11T20:15:25,759 DEBUG [RS:0;9fe0640122ec:41281 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9fe0640122ec:0, corePoolSize=3, maxPoolSize=3
2024-12-11T20:15:25,759 DEBUG [RS:0;9fe0640122ec:41281 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9fe0640122ec:0, corePoolSize=3, maxPoolSize=3
2024-12-11T20:15:25,759 INFO  [RS:0;9fe0640122ec:41281 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-11T20:15:25,759 INFO  [RS:0;9fe0640122ec:41281 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-11T20:15:25,759 INFO  [RS:0;9fe0640122ec:41281 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-11T20:15:25,759 INFO  [RS:0;9fe0640122ec:41281 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-11T20:15:25,759 INFO  [RS:0;9fe0640122ec:41281 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-11T20:15:25,759 INFO  [RS:0;9fe0640122ec:41281 {}] hbase.ChoreService(168): Chore ScheduledChore name=9fe0640122ec,41281,1733948125111-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-11T20:15:25,774 INFO  [RS:0;9fe0640122ec:41281 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-11T20:15:25,774 INFO  [RS:0;9fe0640122ec:41281 {}] hbase.ChoreService(168): Chore ScheduledChore name=9fe0640122ec,41281,1733948125111-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-11T20:15:25,774 INFO  [RS:0;9fe0640122ec:41281 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T20:15:25,774 INFO  [RS:0;9fe0640122ec:41281 {}] regionserver.Replication(171): 9fe0640122ec,41281,1733948125111 started
2024-12-11T20:15:25,788 INFO  [RS:0;9fe0640122ec:41281 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T20:15:25,788 INFO  [RS:0;9fe0640122ec:41281 {}] regionserver.HRegionServer(1482): Serving as 9fe0640122ec,41281,1733948125111, RpcServer on 9fe0640122ec/172.17.0.2:41281, sessionid=0x100cafac1fa0001
2024-12-11T20:15:25,788 DEBUG [RS:0;9fe0640122ec:41281 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-11T20:15:25,788 DEBUG [RS:0;9fe0640122ec:41281 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9fe0640122ec,41281,1733948125111
2024-12-11T20:15:25,788 DEBUG [RS:0;9fe0640122ec:41281 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9fe0640122ec,41281,1733948125111'
2024-12-11T20:15:25,788 DEBUG [RS:0;9fe0640122ec:41281 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-11T20:15:25,789 DEBUG [RS:0;9fe0640122ec:41281 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-11T20:15:25,789 DEBUG [RS:0;9fe0640122ec:41281 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-11T20:15:25,789 DEBUG [RS:0;9fe0640122ec:41281 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-11T20:15:25,789 DEBUG [RS:0;9fe0640122ec:41281 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9fe0640122ec,41281,1733948125111
2024-12-11T20:15:25,789 DEBUG [RS:0;9fe0640122ec:41281 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9fe0640122ec,41281,1733948125111'
2024-12-11T20:15:25,789 DEBUG [RS:0;9fe0640122ec:41281 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-11T20:15:25,789 DEBUG [RS:0;9fe0640122ec:41281 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-11T20:15:25,790 DEBUG [RS:0;9fe0640122ec:41281 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-11T20:15:25,790 INFO  [RS:0;9fe0640122ec:41281 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-11T20:15:25,790 INFO  [RS:0;9fe0640122ec:41281 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-11T20:15:25,848 WARN  [9fe0640122ec:36929 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-12-11T20:15:25,891 INFO  [RS:0;9fe0640122ec:41281 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9fe0640122ec%2C41281%2C1733948125111, suffix=, logDir=hdfs://localhost:37155/user/jenkins/test-data/beeabada-bbfd-e2a4-53bf-1c2a18b752dc/WALs/9fe0640122ec,41281,1733948125111, archiveDir=hdfs://localhost:37155/user/jenkins/test-data/beeabada-bbfd-e2a4-53bf-1c2a18b752dc/oldWALs, maxLogs=32
2024-12-11T20:15:25,892 INFO  [RS:0;9fe0640122ec:41281 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 9fe0640122ec%2C41281%2C1733948125111.1733948125892
2024-12-11T20:15:25,897 INFO  [RS:0;9fe0640122ec:41281 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/beeabada-bbfd-e2a4-53bf-1c2a18b752dc/WALs/9fe0640122ec,41281,1733948125111/9fe0640122ec%2C41281%2C1733948125111.1733948125892
2024-12-11T20:15:25,898 DEBUG [RS:0;9fe0640122ec:41281 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45645:45645),(127.0.0.1/127.0.0.1:34263:34263)]
2024-12-11T20:15:26,098 DEBUG [9fe0640122ec:36929 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-11T20:15:26,098 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=9fe0640122ec,41281,1733948125111
2024-12-11T20:15:26,100 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9fe0640122ec,41281,1733948125111, state=OPENING
2024-12-11T20:15:26,101 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-11T20:15:26,102 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36929-0x100cafac1fa0000, quorum=127.0.0.1:56007, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:15:26,102 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41281-0x100cafac1fa0001, quorum=127.0.0.1:56007, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:15:26,102 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-11T20:15:26,103 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-11T20:15:26,103 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-11T20:15:26,103 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=9fe0640122ec,41281,1733948125111}]
2024-12-11T20:15:26,255 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-11T20:15:26,257 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-17-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38489, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-11T20:15:26,260 INFO  [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-12-11T20:15:26,260 INFO  [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-11T20:15:26,262 INFO  [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9fe0640122ec%2C41281%2C1733948125111.meta, suffix=.meta, logDir=hdfs://localhost:37155/user/jenkins/test-data/beeabada-bbfd-e2a4-53bf-1c2a18b752dc/WALs/9fe0640122ec,41281,1733948125111, archiveDir=hdfs://localhost:37155/user/jenkins/test-data/beeabada-bbfd-e2a4-53bf-1c2a18b752dc/oldWALs, maxLogs=32
2024-12-11T20:15:26,262 INFO  [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 9fe0640122ec%2C41281%2C1733948125111.meta.1733948126262.meta
2024-12-11T20:15:26,270 INFO  [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/beeabada-bbfd-e2a4-53bf-1c2a18b752dc/WALs/9fe0640122ec,41281,1733948125111/9fe0640122ec%2C41281%2C1733948125111.meta.1733948126262.meta
2024-12-11T20:15:26,275 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45645:45645),(127.0.0.1/127.0.0.1:34263:34263)]
2024-12-11T20:15:26,277 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-11T20:15:26,277 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-11T20:15:26,277 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-11T20:15:26,277 INFO  [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-11T20:15:26,277 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-11T20:15:26,277 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-11T20:15:26,277 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-12-11T20:15:26,277 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-12-11T20:15:26,280 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-11T20:15:26,280 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-11T20:15:26,280 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:15:26,281 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T20:15:26,281 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-11T20:15:26,282 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-11T20:15:26,282 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:15:26,282 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T20:15:26,282 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-11T20:15:26,283 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-11T20:15:26,283 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:15:26,283 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T20:15:26,283 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-11T20:15:26,284 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-11T20:15:26,284 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-11T20:15:26,284 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-11T20:15:26,284 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-11T20:15:26,285 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37155/user/jenkins/test-data/beeabada-bbfd-e2a4-53bf-1c2a18b752dc/data/hbase/meta/1588230740
2024-12-11T20:15:26,285 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37155/user/jenkins/test-data/beeabada-bbfd-e2a4-53bf-1c2a18b752dc/data/hbase/meta/1588230740
2024-12-11T20:15:26,286 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-11T20:15:26,286 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-11T20:15:26,287 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-11T20:15:26,288 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-11T20:15:26,289 INFO  [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=841852, jitterRate=0.0704709142446518}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-11T20:15:26,289 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-12-11T20:15:26,289 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1733948126278Writing region info on filesystem at 1733948126278Initializing all the Stores at 1733948126278Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733948126278Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733948126280 (+2 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733948126280Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733948126280Cleaning up temporary data from old regions at 1733948126286 (+6 ms)Running coprocessor post-open hooks at 1733948126289 (+3 ms)Region opened successfully at 1733948126289
2024-12-11T20:15:26,290 INFO  [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733948126255
2024-12-11T20:15:26,292 DEBUG [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-11T20:15:26,292 INFO  [RS_OPEN_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-12-11T20:15:26,293 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=9fe0640122ec,41281,1733948125111
2024-12-11T20:15:26,293 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9fe0640122ec,41281,1733948125111, state=OPEN
2024-12-11T20:15:26,297 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41281-0x100cafac1fa0001, quorum=127.0.0.1:56007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-11T20:15:26,297 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36929-0x100cafac1fa0000, quorum=127.0.0.1:56007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-11T20:15:26,297 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=9fe0640122ec,41281,1733948125111
2024-12-11T20:15:26,297 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-11T20:15:26,297 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-11T20:15:26,299 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-12-11T20:15:26,299 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9fe0640122ec,41281,1733948125111 in 194 msec
2024-12-11T20:15:26,301 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-12-11T20:15:26,301 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 604 msec
2024-12-11T20:15:26,302 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-11T20:15:26,302 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-12-11T20:15:26,303 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-11T20:15:26,303 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9fe0640122ec,41281,1733948125111, seqNum=-1]
2024-12-11T20:15:26,303 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-11T20:15:26,304 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-17-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45153, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-11T20:15:26,309 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 649 msec
2024-12-11T20:15:26,309 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733948126309, completionTime=-1
2024-12-11T20:15:26,309 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-11T20:15:26,309 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-12-11T20:15:26,310 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-12-11T20:15:26,310 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733948186310
2024-12-11T20:15:26,310 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733948246310
2024-12-11T20:15:26,310 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 1 msec
2024-12-11T20:15:26,311 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9fe0640122ec,36929,1733948125065-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-11T20:15:26,311 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9fe0640122ec,36929,1733948125065-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T20:15:26,311 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9fe0640122ec,36929,1733948125065-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T20:15:26,311 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-9fe0640122ec:36929, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T20:15:26,311 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-11T20:15:26,311 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-12-11T20:15:26,312 DEBUG [master/9fe0640122ec:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-12-11T20:15:26,314 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.168sec
2024-12-11T20:15:26,314 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-11T20:15:26,314 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-11T20:15:26,314 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-11T20:15:26,314 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-11T20:15:26,314 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-11T20:15:26,314 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9fe0640122ec,36929,1733948125065-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-11T20:15:26,314 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9fe0640122ec,36929,1733948125065-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-11T20:15:26,316 DEBUG [master/9fe0640122ec:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-12-11T20:15:26,316 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-11T20:15:26,316 INFO  [master/9fe0640122ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9fe0640122ec,36929,1733948125065-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-11T20:15:26,330 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c6abd2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-11T20:15:26,330 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9fe0640122ec,36929,-1 for getting cluster id
2024-12-11T20:15:26,330 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-12-11T20:15:26,332 DEBUG [HMaster-EventLoopGroup-16-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '91817917-8452-4028-9583-c6636f95f57b'
2024-12-11T20:15:26,332 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-12-11T20:15:26,332 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "91817917-8452-4028-9583-c6636f95f57b"

2024-12-11T20:15:26,333 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73a36ed7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-11T20:15:26,333 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9fe0640122ec,36929,-1]
2024-12-11T20:15:26,333 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-12-11T20:15:26,333 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T20:15:26,334 INFO  [HMaster-EventLoopGroup-16-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34546, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-12-11T20:15:26,335 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b2e282d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-11T20:15:26,335 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-11T20:15:26,336 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9fe0640122ec,41281,1733948125111, seqNum=-1]
2024-12-11T20:15:26,336 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-11T20:15:26,337 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-17-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54098, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-11T20:15:26,338 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=9fe0640122ec,36929,1733948125065
2024-12-11T20:15:26,339 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-11T20:15:26,341 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-12-11T20:15:26,341 INFO  [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-11T20:15:26,342 INFO  [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=test.com%2C8080%2C1, suffix=, logDir=hdfs://localhost:37155/user/jenkins/test-data/beeabada-bbfd-e2a4-53bf-1c2a18b752dc/WALs/test.com,8080,1, archiveDir=hdfs://localhost:37155/user/jenkins/test-data/beeabada-bbfd-e2a4-53bf-1c2a18b752dc/oldWALs, maxLogs=32
2024-12-11T20:15:26,343 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1733948126343
2024-12-11T20:15:26,347 INFO  [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/beeabada-bbfd-e2a4-53bf-1c2a18b752dc/WALs/test.com,8080,1/test.com%2C8080%2C1.1733948126343
2024-12-11T20:15:26,348 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34263:34263),(127.0.0.1/127.0.0.1:45645:45645)]
2024-12-11T20:15:26,349 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1733948126349
2024-12-11T20:15:26,352 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42593,1733947942024/9fe0640122ec%2C42593%2C1733947942024.meta.1733947942936.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:15:26,352 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40619/user/jenkins/test-data/b554e755-7d91-19ed-0263-202eda75055f/WALs/9fe0640122ec,42419,1733947943179/9fe0640122ec%2C42419%2C1733947943179.1733947943403
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-11T20:15:26,354 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:15:26,354 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:15:26,354 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:15:26,354 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:15:26,354 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:15:26,354 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/beeabada-bbfd-e2a4-53bf-1c2a18b752dc/WALs/test.com,8080,1/test.com%2C8080%2C1.1733948126343 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/beeabada-bbfd-e2a4-53bf-1c2a18b752dc/WALs/test.com,8080,1/test.com%2C8080%2C1.1733948126349
2024-12-11T20:15:26,355 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34263:34263),(127.0.0.1/127.0.0.1:45645:45645)]
2024-12-11T20:15:26,355 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:37155/user/jenkins/test-data/beeabada-bbfd-e2a4-53bf-1c2a18b752dc/WALs/test.com,8080,1/test.com%2C8080%2C1.1733948126343 is not closed yet, will try archiving it next time
2024-12-11T20:15:26,355 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:15:26,355 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:15:26,356 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33713 is added to blk_1073741835_1011 (size=93)
2024-12-11T20:15:26,356 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43549 is added to blk_1073741835_1011 (size=93)
2024-12-11T20:15:26,357 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:37155/user/jenkins/test-data/beeabada-bbfd-e2a4-53bf-1c2a18b752dc/WALs/test.com,8080,1/test.com%2C8080%2C1.1733948126343 to hdfs://localhost:37155/user/jenkins/test-data/beeabada-bbfd-e2a4-53bf-1c2a18b752dc/oldWALs/test.com%2C8080%2C1.1733948126343
2024-12-11T20:15:26,357 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:15:26,357 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:15:26,357 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:15:26,358 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33713 is added to blk_1073741836_1012 (size=93)
2024-12-11T20:15:26,358 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43549 is added to blk_1073741836_1012 (size=93)
2024-12-11T20:15:26,360 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/beeabada-bbfd-e2a4-53bf-1c2a18b752dc/oldWALs
2024-12-11T20:15:26,360 INFO  [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog test.com%2C8080%2C1:(num 1733948126349)
2024-12-11T20:15:26,360 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-12-11T20:15:26,360 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-11T20:15:26,360 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-11T20:15:26,360 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T20:15:26,361 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-12-11T20:15:26,361 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T20:15:26,361 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-11T20:15:26,361 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=983937006, stopped=false
2024-12-11T20:15:26,361 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=9fe0640122ec,36929,1733948125065
2024-12-11T20:15:26,363 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36929-0x100cafac1fa0000, quorum=127.0.0.1:56007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-11T20:15:26,363 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41281-0x100cafac1fa0001, quorum=127.0.0.1:56007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-11T20:15:26,363 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36929-0x100cafac1fa0000, quorum=127.0.0.1:56007, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:15:26,363 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41281-0x100cafac1fa0001, quorum=127.0.0.1:56007, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:15:26,363 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-11T20:15:26,363 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-11T20:15:26,363 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-11T20:15:26,363 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T20:15:26,363 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9fe0640122ec,41281,1733948125111' *****
2024-12-11T20:15:26,363 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-12-11T20:15:26,363 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41281-0x100cafac1fa0001, quorum=127.0.0.1:56007, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-11T20:15:26,363 INFO  [RS:0;9fe0640122ec:41281 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-11T20:15:26,363 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36929-0x100cafac1fa0000, quorum=127.0.0.1:56007, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-11T20:15:26,363 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-12-11T20:15:26,364 INFO  [RS:0;9fe0640122ec:41281 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-11T20:15:26,364 INFO  [RS:0;9fe0640122ec:41281 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-11T20:15:26,364 INFO  [RS:0;9fe0640122ec:41281 {}] regionserver.HRegionServer(959): stopping server 9fe0640122ec,41281,1733948125111
2024-12-11T20:15:26,364 INFO  [RS:0;9fe0640122ec:41281 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-11T20:15:26,364 INFO  [RS:0;9fe0640122ec:41281 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;9fe0640122ec:41281.
2024-12-11T20:15:26,364 DEBUG [RS:0;9fe0640122ec:41281 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-11T20:15:26,364 DEBUG [RS:0;9fe0640122ec:41281 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T20:15:26,364 INFO  [RS:0;9fe0640122ec:41281 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-11T20:15:26,364 INFO  [RS:0;9fe0640122ec:41281 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-11T20:15:26,364 INFO  [RS:0;9fe0640122ec:41281 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-11T20:15:26,364 INFO  [RS:0;9fe0640122ec:41281 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-12-11T20:15:26,364 INFO  [RS:0;9fe0640122ec:41281 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close
2024-12-11T20:15:26,364 DEBUG [RS:0;9fe0640122ec:41281 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740}
2024-12-11T20:15:26,365 DEBUG [RS:0;9fe0640122ec:41281 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-12-11T20:15:26,365 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-11T20:15:26,365 INFO  [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-11T20:15:26,365 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-11T20:15:26,365 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-11T20:15:26,365 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-11T20:15:26,365 INFO  [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=74 B heapSize=1.22 KB
2024-12-11T20:15:26,387 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37155/user/jenkins/test-data/beeabada-bbfd-e2a4-53bf-1c2a18b752dc/data/hbase/meta/1588230740/.tmp/ns/46c04744fd224e57a04f276888957627 is 43, key is default/ns:d/1733948126305/Put/seqid=0
2024-12-11T20:15:26,392 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43549 is added to blk_1073741837_1013 (size=5153)
2024-12-11T20:15:26,392 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33713 is added to blk_1073741837_1013 (size=5153)
2024-12-11T20:15:26,392 INFO  [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:37155/user/jenkins/test-data/beeabada-bbfd-e2a4-53bf-1c2a18b752dc/data/hbase/meta/1588230740/.tmp/ns/46c04744fd224e57a04f276888957627
2024-12-11T20:15:26,397 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37155/user/jenkins/test-data/beeabada-bbfd-e2a4-53bf-1c2a18b752dc/data/hbase/meta/1588230740/.tmp/ns/46c04744fd224e57a04f276888957627 as hdfs://localhost:37155/user/jenkins/test-data/beeabada-bbfd-e2a4-53bf-1c2a18b752dc/data/hbase/meta/1588230740/ns/46c04744fd224e57a04f276888957627
2024-12-11T20:15:26,401 INFO  [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37155/user/jenkins/test-data/beeabada-bbfd-e2a4-53bf-1c2a18b752dc/data/hbase/meta/1588230740/ns/46c04744fd224e57a04f276888957627, entries=2, sequenceid=6, filesize=5.0 K
2024-12-11T20:15:26,402 INFO  [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 37ms, sequenceid=6, compaction requested=false
2024-12-11T20:15:26,402 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-12-11T20:15:26,406 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37155/user/jenkins/test-data/beeabada-bbfd-e2a4-53bf-1c2a18b752dc/data/hbase/meta/1588230740/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1
2024-12-11T20:15:26,407 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-11T20:15:26,407 INFO  [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-11T20:15:26,407 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733948126365Running coprocessor pre-close hooks at 1733948126365Disabling compacts and flushes for region at 1733948126365Disabling writes for close at 1733948126365Obtaining lock to block concurrent updates at 1733948126365Preparing flush snapshotting stores in 1588230740 at 1733948126365Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=74, getHeapSize=1184, getOffHeapSize=0, getCellsCount=2 at 1733948126365Flushing stores of hbase:meta,,1.1588230740 at 1733948126366 (+1 ms)Flushing 1588230740/ns: creating writer at 1733948126366Flushing 1588230740/ns: appending metadata at 1733948126387 (+21 ms)Flushing 1588230740/ns: closing flushed file at 1733948126387Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@191aef17: reopening flushed file at 1733948126397 (+10 ms)Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 37ms, sequenceid=6, compaction requested=false at 1733948126402 (+5 ms)Writing region close event to WAL at 1733948126403 (+1 ms)Running coprocessor post-close hooks at 1733948126407 (+4 ms)Closed at 1733948126407
2024-12-11T20:15:26,407 DEBUG [RS_CLOSE_META-regionserver/9fe0640122ec:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-11T20:15:26,565 INFO  [RS:0;9fe0640122ec:41281 {}] regionserver.HRegionServer(976): stopping server 9fe0640122ec,41281,1733948125111; all regions closed.
2024-12-11T20:15:26,565 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:15:26,565 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:15:26,566 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:15:26,566 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:15:26,566 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:15:26,568 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43549 is added to blk_1073741834_1010 (size=1152)
2024-12-11T20:15:26,568 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33713 is added to blk_1073741834_1010 (size=1152)
2024-12-11T20:15:26,570 DEBUG [RS:0;9fe0640122ec:41281 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/beeabada-bbfd-e2a4-53bf-1c2a18b752dc/oldWALs
2024-12-11T20:15:26,570 INFO  [RS:0;9fe0640122ec:41281 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 9fe0640122ec%2C41281%2C1733948125111.meta:.meta(num 1733948126262)
2024-12-11T20:15:26,571 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:15:26,571 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:15:26,571 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:15:26,571 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:15:26,571 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:15:26,573 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43549 is added to blk_1073741833_1009 (size=93)
2024-12-11T20:15:26,573 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33713 is added to blk_1073741833_1009 (size=93)
2024-12-11T20:15:26,575 DEBUG [RS:0;9fe0640122ec:41281 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/beeabada-bbfd-e2a4-53bf-1c2a18b752dc/oldWALs
2024-12-11T20:15:26,575 INFO  [RS:0;9fe0640122ec:41281 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 9fe0640122ec%2C41281%2C1733948125111:(num 1733948125892)
2024-12-11T20:15:26,575 DEBUG [RS:0;9fe0640122ec:41281 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-11T20:15:26,575 INFO  [RS:0;9fe0640122ec:41281 {}] regionserver.LeaseManager(133): Closed leases
2024-12-11T20:15:26,575 INFO  [RS:0;9fe0640122ec:41281 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-11T20:15:26,575 INFO  [RS:0;9fe0640122ec:41281 {}] hbase.ChoreService(370): Chore service for: regionserver/9fe0640122ec:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown
2024-12-11T20:15:26,575 INFO  [RS:0;9fe0640122ec:41281 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-11T20:15:26,575 INFO  [regionserver/9fe0640122ec:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-11T20:15:26,576 INFO  [RS:0;9fe0640122ec:41281 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41281
2024-12-11T20:15:26,578 INFO  [RS:0;9fe0640122ec:41281 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-11T20:15:26,578 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36929-0x100cafac1fa0000, quorum=127.0.0.1:56007, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-11T20:15:26,578 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41281-0x100cafac1fa0001, quorum=127.0.0.1:56007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9fe0640122ec,41281,1733948125111
2024-12-11T20:15:26,579 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9fe0640122ec,41281,1733948125111]
2024-12-11T20:15:26,580 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9fe0640122ec,41281,1733948125111 already deleted, retry=false
2024-12-11T20:15:26,580 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 9fe0640122ec,41281,1733948125111 expired; onlineServers=0
2024-12-11T20:15:26,580 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '9fe0640122ec,36929,1733948125065' *****
2024-12-11T20:15:26,580 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-11T20:15:26,580 INFO  [M:0;9fe0640122ec:36929 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-11T20:15:26,580 INFO  [M:0;9fe0640122ec:36929 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-11T20:15:26,581 DEBUG [M:0;9fe0640122ec:36929 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-11T20:15:26,581 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-11T20:15:26,581 DEBUG [M:0;9fe0640122ec:36929 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-11T20:15:26,581 DEBUG [master/9fe0640122ec:0:becomeActiveMaster-HFileCleaner.large.0-1733948125665 {}] cleaner.HFileCleaner(306): Exit Thread[master/9fe0640122ec:0:becomeActiveMaster-HFileCleaner.large.0-1733948125665,5,FailOnTimeoutGroup]
2024-12-11T20:15:26,581 DEBUG [master/9fe0640122ec:0:becomeActiveMaster-HFileCleaner.small.0-1733948125667 {}] cleaner.HFileCleaner(306): Exit Thread[master/9fe0640122ec:0:becomeActiveMaster-HFileCleaner.small.0-1733948125667,5,FailOnTimeoutGroup]
2024-12-11T20:15:26,581 INFO  [M:0;9fe0640122ec:36929 {}] hbase.ChoreService(370): Chore service for: master/9fe0640122ec:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-12-11T20:15:26,581 INFO  [M:0;9fe0640122ec:36929 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-11T20:15:26,581 DEBUG [M:0;9fe0640122ec:36929 {}] master.HMaster(1795): Stopping service threads
2024-12-11T20:15:26,581 INFO  [M:0;9fe0640122ec:36929 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-11T20:15:26,581 INFO  [M:0;9fe0640122ec:36929 {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-11T20:15:26,581 INFO  [M:0;9fe0640122ec:36929 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-11T20:15:26,581 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-11T20:15:26,582 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36929-0x100cafac1fa0000, quorum=127.0.0.1:56007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-11T20:15:26,583 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36929-0x100cafac1fa0000, quorum=127.0.0.1:56007, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-11T20:15:26,583 DEBUG [M:0;9fe0640122ec:36929 {}] zookeeper.ZKUtil(347): master:36929-0x100cafac1fa0000, quorum=127.0.0.1:56007, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-11T20:15:26,583 WARN  [M:0;9fe0640122ec:36929 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-11T20:15:26,583 INFO  [M:0;9fe0640122ec:36929 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:37155/user/jenkins/test-data/beeabada-bbfd-e2a4-53bf-1c2a18b752dc/.lastflushedseqids
2024-12-11T20:15:26,588 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43549 is added to blk_1073741838_1014 (size=108)
2024-12-11T20:15:26,588 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33713 is added to blk_1073741838_1014 (size=108)
2024-12-11T20:15:26,589 INFO  [M:0;9fe0640122ec:36929 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-12-11T20:15:26,589 INFO  [M:0;9fe0640122ec:36929 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-11T20:15:26,589 DEBUG [M:0;9fe0640122ec:36929 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-11T20:15:26,589 INFO  [M:0;9fe0640122ec:36929 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T20:15:26,589 DEBUG [M:0;9fe0640122ec:36929 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T20:15:26,589 DEBUG [M:0;9fe0640122ec:36929 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-11T20:15:26,589 DEBUG [M:0;9fe0640122ec:36929 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T20:15:26,589 INFO  [M:0;9fe0640122ec:36929 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.67 KB heapSize=11.34 KB
2024-12-11T20:15:26,605 DEBUG [M:0;9fe0640122ec:36929 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37155/user/jenkins/test-data/beeabada-bbfd-e2a4-53bf-1c2a18b752dc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/31a633f49d4f4a6794775f0b82c7d1dd is 82, key is hbase:meta,,1/info:regioninfo/1733948126293/Put/seqid=0
2024-12-11T20:15:26,609 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33713 is added to blk_1073741839_1015 (size=5672)
2024-12-11T20:15:26,610 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43549 is added to blk_1073741839_1015 (size=5672)
2024-12-11T20:15:26,610 INFO  [M:0;9fe0640122ec:36929 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:37155/user/jenkins/test-data/beeabada-bbfd-e2a4-53bf-1c2a18b752dc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/31a633f49d4f4a6794775f0b82c7d1dd
2024-12-11T20:15:26,629 DEBUG [M:0;9fe0640122ec:36929 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37155/user/jenkins/test-data/beeabada-bbfd-e2a4-53bf-1c2a18b752dc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c2c163ce482a47c7a765386a6f3472c8 is 240, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1733948126308/Put/seqid=0
2024-12-11T20:15:26,633 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43549 is added to blk_1073741840_1016 (size=5275)
2024-12-11T20:15:26,633 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33713 is added to blk_1073741840_1016 (size=5275)
2024-12-11T20:15:26,634 INFO  [M:0;9fe0640122ec:36929 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:37155/user/jenkins/test-data/beeabada-bbfd-e2a4-53bf-1c2a18b752dc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c2c163ce482a47c7a765386a6f3472c8
2024-12-11T20:15:26,654 DEBUG [M:0;9fe0640122ec:36929 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37155/user/jenkins/test-data/beeabada-bbfd-e2a4-53bf-1c2a18b752dc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2dfceb3087074854b41d3fb279447de7 is 69, key is 9fe0640122ec,41281,1733948125111/rs:state/1733948125749/Put/seqid=0
2024-12-11T20:15:26,658 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43549 is added to blk_1073741841_1017 (size=5156)
2024-12-11T20:15:26,658 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33713 is added to blk_1073741841_1017 (size=5156)
2024-12-11T20:15:26,659 INFO  [M:0;9fe0640122ec:36929 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:37155/user/jenkins/test-data/beeabada-bbfd-e2a4-53bf-1c2a18b752dc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2dfceb3087074854b41d3fb279447de7
2024-12-11T20:15:26,677 DEBUG [M:0;9fe0640122ec:36929 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37155/user/jenkins/test-data/beeabada-bbfd-e2a4-53bf-1c2a18b752dc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/3cfd4207c80e44f490e8ed577731f977 is 52, key is load_balancer_on/state:d/1733948126340/Put/seqid=0
2024-12-11T20:15:26,679 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41281-0x100cafac1fa0001, quorum=127.0.0.1:56007, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-11T20:15:26,679 INFO  [RS:0;9fe0640122ec:41281 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-11T20:15:26,679 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41281-0x100cafac1fa0001, quorum=127.0.0.1:56007, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-11T20:15:26,679 INFO  [RS:0;9fe0640122ec:41281 {}] regionserver.HRegionServer(1031): Exiting; stopping=9fe0640122ec,41281,1733948125111; zookeeper connection closed.
2024-12-11T20:15:26,680 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6a93e23a {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6a93e23a
2024-12-11T20:15:26,680 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-12-11T20:15:26,682 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43549 is added to blk_1073741842_1018 (size=5056)
2024-12-11T20:15:26,682 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33713 is added to blk_1073741842_1018 (size=5056)
2024-12-11T20:15:26,682 INFO  [M:0;9fe0640122ec:36929 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:37155/user/jenkins/test-data/beeabada-bbfd-e2a4-53bf-1c2a18b752dc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/3cfd4207c80e44f490e8ed577731f977
2024-12-11T20:15:26,686 DEBUG [M:0;9fe0640122ec:36929 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37155/user/jenkins/test-data/beeabada-bbfd-e2a4-53bf-1c2a18b752dc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/31a633f49d4f4a6794775f0b82c7d1dd as hdfs://localhost:37155/user/jenkins/test-data/beeabada-bbfd-e2a4-53bf-1c2a18b752dc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/31a633f49d4f4a6794775f0b82c7d1dd
2024-12-11T20:15:26,690 INFO  [M:0;9fe0640122ec:36929 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37155/user/jenkins/test-data/beeabada-bbfd-e2a4-53bf-1c2a18b752dc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/31a633f49d4f4a6794775f0b82c7d1dd, entries=8, sequenceid=29, filesize=5.5 K
2024-12-11T20:15:26,691 DEBUG [M:0;9fe0640122ec:36929 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37155/user/jenkins/test-data/beeabada-bbfd-e2a4-53bf-1c2a18b752dc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c2c163ce482a47c7a765386a6f3472c8 as hdfs://localhost:37155/user/jenkins/test-data/beeabada-bbfd-e2a4-53bf-1c2a18b752dc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c2c163ce482a47c7a765386a6f3472c8
2024-12-11T20:15:26,694 INFO  [M:0;9fe0640122ec:36929 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37155/user/jenkins/test-data/beeabada-bbfd-e2a4-53bf-1c2a18b752dc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c2c163ce482a47c7a765386a6f3472c8, entries=3, sequenceid=29, filesize=5.2 K
2024-12-11T20:15:26,695 DEBUG [M:0;9fe0640122ec:36929 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37155/user/jenkins/test-data/beeabada-bbfd-e2a4-53bf-1c2a18b752dc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2dfceb3087074854b41d3fb279447de7 as hdfs://localhost:37155/user/jenkins/test-data/beeabada-bbfd-e2a4-53bf-1c2a18b752dc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2dfceb3087074854b41d3fb279447de7
2024-12-11T20:15:26,699 INFO  [M:0;9fe0640122ec:36929 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37155/user/jenkins/test-data/beeabada-bbfd-e2a4-53bf-1c2a18b752dc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2dfceb3087074854b41d3fb279447de7, entries=1, sequenceid=29, filesize=5.0 K
2024-12-11T20:15:26,699 DEBUG [M:0;9fe0640122ec:36929 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37155/user/jenkins/test-data/beeabada-bbfd-e2a4-53bf-1c2a18b752dc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/3cfd4207c80e44f490e8ed577731f977 as hdfs://localhost:37155/user/jenkins/test-data/beeabada-bbfd-e2a4-53bf-1c2a18b752dc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/3cfd4207c80e44f490e8ed577731f977
2024-12-11T20:15:26,703 INFO  [M:0;9fe0640122ec:36929 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37155/user/jenkins/test-data/beeabada-bbfd-e2a4-53bf-1c2a18b752dc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/3cfd4207c80e44f490e8ed577731f977, entries=1, sequenceid=29, filesize=4.9 K
2024-12-11T20:15:26,704 INFO  [M:0;9fe0640122ec:36929 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 115ms, sequenceid=29, compaction requested=false
2024-12-11T20:15:26,706 INFO  [M:0;9fe0640122ec:36929 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-11T20:15:26,706 DEBUG [M:0;9fe0640122ec:36929 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733948126589Disabling compacts and flushes for region at 1733948126589Disabling writes for close at 1733948126589Obtaining lock to block concurrent updates at 1733948126589Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733948126589Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7850, getHeapSize=11544, getOffHeapSize=0, getCellsCount=36 at 1733948126590 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733948126590Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733948126590Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733948126605 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733948126605Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733948126614 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733948126629 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733948126629Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733948126638 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733948126653 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733948126653Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733948126663 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733948126677 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733948126677Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7f70d9cb: reopening flushed file at 1733948126686 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4c2e1018: reopening flushed file at 1733948126690 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6eaee5c1: reopening flushed file at 1733948126694 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@508729b6: reopening flushed file at 1733948126699 (+5 ms)Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 115ms, sequenceid=29, compaction requested=false at 1733948126704 (+5 ms)Writing region close event to WAL at 1733948126706 (+2 ms)Closed at 1733948126706
2024-12-11T20:15:26,706 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:15:26,706 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:15:26,706 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:15:26,707 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:15:26,707 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-11T20:15:26,708 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33713 is added to blk_1073741830_1006 (size=10311)
2024-12-11T20:15:26,709 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43549 is added to blk_1073741830_1006 (size=10311)
2024-12-11T20:15:26,709 INFO  [M:0;9fe0640122ec:36929 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-12-11T20:15:26,709 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-11T20:15:26,709 INFO  [M:0;9fe0640122ec:36929 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36929
2024-12-11T20:15:26,709 INFO  [M:0;9fe0640122ec:36929 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-11T20:15:26,811 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36929-0x100cafac1fa0000, quorum=127.0.0.1:56007, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-11T20:15:26,811 INFO  [M:0;9fe0640122ec:36929 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-11T20:15:26,811 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36929-0x100cafac1fa0000, quorum=127.0.0.1:56007, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-11T20:15:26,814 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2d8770d2{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T20:15:26,814 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@78791d2c{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-11T20:15:26,814 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-11T20:15:26,815 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@392cf3b8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-11T20:15:26,815 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@115f614b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2b1c9106-caf5-ed09-2d13-05be0c65ac5f/hadoop.log.dir/,STOPPED}
2024-12-11T20:15:26,816 WARN  [BP-1466799083-172.17.0.2-1733948124427 heartbeating to localhost/127.0.0.1:37155 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-11T20:15:26,816 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-11T20:15:26,816 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-11T20:15:26,816 WARN  [BP-1466799083-172.17.0.2-1733948124427 heartbeating to localhost/127.0.0.1:37155 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1466799083-172.17.0.2-1733948124427 (Datanode Uuid dd1e72ec-36be-4275-876e-e3d1a433898c) service to localhost/127.0.0.1:37155
2024-12-11T20:15:26,817 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2b1c9106-caf5-ed09-2d13-05be0c65ac5f/cluster_f31c84cd-b048-50a9-4468-390e6aa69c32/data/data3/current/BP-1466799083-172.17.0.2-1733948124427 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T20:15:26,817 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2b1c9106-caf5-ed09-2d13-05be0c65ac5f/cluster_f31c84cd-b048-50a9-4468-390e6aa69c32/data/data4/current/BP-1466799083-172.17.0.2-1733948124427 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T20:15:26,817 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-11T20:15:26,819 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@45e68726{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-11T20:15:26,819 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6e1cda33{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-11T20:15:26,819 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-11T20:15:26,819 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@352a3917{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-11T20:15:26,819 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@536a5c19{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2b1c9106-caf5-ed09-2d13-05be0c65ac5f/hadoop.log.dir/,STOPPED}
2024-12-11T20:15:26,821 WARN  [BP-1466799083-172.17.0.2-1733948124427 heartbeating to localhost/127.0.0.1:37155 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-11T20:15:26,821 WARN  [BP-1466799083-172.17.0.2-1733948124427 heartbeating to localhost/127.0.0.1:37155 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1466799083-172.17.0.2-1733948124427 (Datanode Uuid c5f0568f-755c-49bf-9570-11811c1e5b15) service to localhost/127.0.0.1:37155
2024-12-11T20:15:26,821 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-11T20:15:26,821 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-11T20:15:26,821 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2b1c9106-caf5-ed09-2d13-05be0c65ac5f/cluster_f31c84cd-b048-50a9-4468-390e6aa69c32/data/data1/current/BP-1466799083-172.17.0.2-1733948124427 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T20:15:26,821 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2b1c9106-caf5-ed09-2d13-05be0c65ac5f/cluster_f31c84cd-b048-50a9-4468-390e6aa69c32/data/data2/current/BP-1466799083-172.17.0.2-1733948124427 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-11T20:15:26,822 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-11T20:15:26,827 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@49428dc3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-11T20:15:26,828 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@186dc28{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-11T20:15:26,828 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-11T20:15:26,828 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@246d323a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-11T20:15:26,828 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2d48d0f6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2b1c9106-caf5-ed09-2d13-05be0c65ac5f/hadoop.log.dir/,STOPPED}
2024-12-11T20:15:26,835 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-12-11T20:15:26,849 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-12-11T20:15:26,858 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=277 (was 239)
Potentially hanging thread: LeaseRenewer:jenkins@localhost:37155
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-42-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-45-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:37155 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-45-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-43-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: globalEventExecutor-1-21
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	app//io.netty.util.concurrent.GlobalEventExecutor.takeTask(GlobalEventExecutor.java:113)
	app//io.netty.util.concurrent.GlobalEventExecutor$TaskRunner.run(GlobalEventExecutor.java:259)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-43-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37155
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-43-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:37155 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-44-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-45-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.7@localhost:37155
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-16-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37155
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:37155 from jenkins.hfs.7
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-42-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-44-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37155
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-42-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-16-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-16-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-44-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=538 (was 515) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=65 (was 71), ProcessCount=11 (was 11), AvailableMemoryMB=273 (was 280)