2024-11-24 04:27:51,709 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca
2024-11-24 04:27:51,723 main DEBUG Took 0.012254 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging
2024-11-24 04:27:51,724 main DEBUG PluginManager 'Core' found 129 plugins
2024-11-24 04:27:51,724 main DEBUG PluginManager 'Level' found 0 plugins
2024-11-24 04:27:51,726 main DEBUG PluginManager 'Lookup' found 16 plugins
2024-11-24 04:27:51,727 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-24 04:27:51,734 main DEBUG PluginManager 'TypeConverter' found 26 plugins
2024-11-24 04:27:51,748 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-24 04:27:51,750 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-24 04:27:51,751 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-24 04:27:51,751 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-24 04:27:51,752 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-24 04:27:51,752 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-24 04:27:51,754 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-24 04:27:51,754 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-24 04:27:51,755 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-24 04:27:51,755 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-24 04:27:51,756 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-24 04:27:51,756 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-24 04:27:51,757 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-24 04:27:51,757 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-24 04:27:51,758 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-24 04:27:51,758 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-24 04:27:51,759 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-24 04:27:51,759 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-24 04:27:51,760 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-24 04:27:51,760 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-24 04:27:51,761 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-24 04:27:51,761 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-24 04:27:51,762 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-24 04:27:51,762 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-24 04:27:51,763 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-24 04:27:51,763 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger].
2024-11-24 04:27:51,765 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-24 04:27:51,767 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin].
2024-11-24 04:27:51,769 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root})
2024-11-24 04:27:51,770 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout].
2024-11-24 04:27:51,771 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null")
2024-11-24 04:27:51,771 main DEBUG PluginManager 'Converter' found 47 plugins
2024-11-24 04:27:51,781 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender].
2024-11-24 04:27:51,783 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={})
2024-11-24 04:27:51,785 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR
2024-11-24 04:27:51,785 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin].
2024-11-24 04:27:51,786 main DEBUG createAppenders(={Console})
2024-11-24 04:27:51,787 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca initialized
2024-11-24 04:27:51,787 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca
2024-11-24 04:27:51,788 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca OK.
2024-11-24 04:27:51,788 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1
2024-11-24 04:27:51,789 main DEBUG OutputStream closed
2024-11-24 04:27:51,789 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true
2024-11-24 04:27:51,789 main DEBUG Appender DefaultConsole-1 stopped with status true
2024-11-24 04:27:51,790 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@6404f418 OK
2024-11-24 04:27:51,870 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6
2024-11-24 04:27:51,872 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger
2024-11-24 04:27:51,873 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector
2024-11-24 04:27:51,874 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=
2024-11-24 04:27:51,875 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory
2024-11-24 04:27:51,875 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter
2024-11-24 04:27:51,875 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper
2024-11-24 04:27:51,875 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j
2024-11-24 04:27:51,876 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl
2024-11-24 04:27:51,876 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans
2024-11-24 04:27:51,876 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase
2024-11-24 04:27:51,877 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop
2024-11-24 04:27:51,877 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers
2024-11-24 04:27:51,877 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices
2024-11-24 04:27:51,877 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig
2024-11-24 04:27:51,878 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel
2024-11-24 04:27:51,878 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore
2024-11-24 04:27:51,879 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console
2024-11-24 04:27:51,881 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-11-24 04:27:51,881 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@6dab9b6d) with optional ClassLoader: null
2024-11-24 04:27:51,881 main DEBUG Shutdown hook enabled. Registering a new one.
2024-11-24 04:27:51,882 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@6dab9b6d] started OK.
2024-11-24T04:27:52,183 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/03752be3-12ba-e59c-4567-8f2d22dd32ed
2024-11-24 04:27:52,187 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED)
2024-11-24 04:27:52,188 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-11-24T04:27:52,198 INFO  [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestLogRolling timeout: 13 mins
2024-11-24T04:27:52,235 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=132, ProcessCount=11, AvailableMemoryMB=6961
2024-11-24T04:27:52,238 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-24T04:27:52,252 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/03752be3-12ba-e59c-4567-8f2d22dd32ed/cluster_6735412e-b74b-ee63-bfe9-fcdf79afd6b9, deleteOnExit=true
2024-11-24T04:27:52,252 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-24T04:27:52,253 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/03752be3-12ba-e59c-4567-8f2d22dd32ed/test.cache.data in system properties and HBase conf
2024-11-24T04:27:52,253 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/03752be3-12ba-e59c-4567-8f2d22dd32ed/hadoop.tmp.dir in system properties and HBase conf
2024-11-24T04:27:52,254 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/03752be3-12ba-e59c-4567-8f2d22dd32ed/hadoop.log.dir in system properties and HBase conf
2024-11-24T04:27:52,254 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/03752be3-12ba-e59c-4567-8f2d22dd32ed/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-24T04:27:52,255 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/03752be3-12ba-e59c-4567-8f2d22dd32ed/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-24T04:27:52,255 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-24T04:27:52,346 WARN  [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
2024-11-24T04:27:52,441 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-24T04:27:52,445 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/03752be3-12ba-e59c-4567-8f2d22dd32ed/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-24T04:27:52,445 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/03752be3-12ba-e59c-4567-8f2d22dd32ed/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-24T04:27:52,446 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/03752be3-12ba-e59c-4567-8f2d22dd32ed/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-24T04:27:52,446 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/03752be3-12ba-e59c-4567-8f2d22dd32ed/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-24T04:27:52,447 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/03752be3-12ba-e59c-4567-8f2d22dd32ed/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-24T04:27:52,447 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/03752be3-12ba-e59c-4567-8f2d22dd32ed/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-24T04:27:52,448 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/03752be3-12ba-e59c-4567-8f2d22dd32ed/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-24T04:27:52,448 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/03752be3-12ba-e59c-4567-8f2d22dd32ed/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-24T04:27:52,448 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/03752be3-12ba-e59c-4567-8f2d22dd32ed/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-24T04:27:52,449 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/03752be3-12ba-e59c-4567-8f2d22dd32ed/nfs.dump.dir in system properties and HBase conf
2024-11-24T04:27:52,449 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/03752be3-12ba-e59c-4567-8f2d22dd32ed/java.io.tmpdir in system properties and HBase conf
2024-11-24T04:27:52,449 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/03752be3-12ba-e59c-4567-8f2d22dd32ed/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-24T04:27:52,450 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/03752be3-12ba-e59c-4567-8f2d22dd32ed/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-24T04:27:52,450 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/03752be3-12ba-e59c-4567-8f2d22dd32ed/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-24T04:27:52,917 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-24T04:27:53,245 WARN  [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
2024-11-24T04:27:53,322 INFO  [Time-limited test {}] log.Log(170): Logging initialized @2382ms to org.eclipse.jetty.util.log.Slf4jLog
2024-11-24T04:27:53,398 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T04:27:53,461 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-24T04:27:53,482 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-24T04:27:53,482 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-24T04:27:53,483 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-24T04:27:53,496 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T04:27:53,498 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@ddc8467{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/03752be3-12ba-e59c-4567-8f2d22dd32ed/hadoop.log.dir/,AVAILABLE}
2024-11-24T04:27:53,499 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@70be1389{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-24T04:27:53,697 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@735fa16a{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/03752be3-12ba-e59c-4567-8f2d22dd32ed/java.io.tmpdir/jetty-localhost-45559-hadoop-hdfs-3_4_1-tests_jar-_-any-15010099031956450318/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-24T04:27:53,706 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6c26a5a3{HTTP/1.1, (http/1.1)}{localhost:45559}
2024-11-24T04:27:53,707 INFO  [Time-limited test {}] server.Server(415): Started @2767ms
2024-11-24T04:27:53,739 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-24T04:27:54,108 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T04:27:54,115 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-24T04:27:54,116 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-24T04:27:54,116 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-24T04:27:54,116 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-24T04:27:54,117 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@28778f0f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/03752be3-12ba-e59c-4567-8f2d22dd32ed/hadoop.log.dir/,AVAILABLE}
2024-11-24T04:27:54,117 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@371e191c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-24T04:27:54,241 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7b07d1ba{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/03752be3-12ba-e59c-4567-8f2d22dd32ed/java.io.tmpdir/jetty-localhost-35895-hadoop-hdfs-3_4_1-tests_jar-_-any-15421740014641300974/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T04:27:54,241 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@43e0a762{HTTP/1.1, (http/1.1)}{localhost:35895}
2024-11-24T04:27:54,242 INFO  [Time-limited test {}] server.Server(415): Started @3303ms
2024-11-24T04:27:54,303 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-24T04:27:54,427 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T04:27:54,432 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-24T04:27:54,438 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-24T04:27:54,438 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-24T04:27:54,439 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-24T04:27:54,440 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@11effdcd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/03752be3-12ba-e59c-4567-8f2d22dd32ed/hadoop.log.dir/,AVAILABLE}
2024-11-24T04:27:54,440 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2d48d695{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-24T04:27:54,560 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1bf97579{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/03752be3-12ba-e59c-4567-8f2d22dd32ed/java.io.tmpdir/jetty-localhost-45339-hadoop-hdfs-3_4_1-tests_jar-_-any-9999770472300959026/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T04:27:54,562 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@22b88bcb{HTTP/1.1, (http/1.1)}{localhost:45339}
2024-11-24T04:27:54,562 INFO  [Time-limited test {}] server.Server(415): Started @3623ms
2024-11-24T04:27:54,565 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-24T04:27:54,765 WARN  [Thread-97 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/03752be3-12ba-e59c-4567-8f2d22dd32ed/cluster_6735412e-b74b-ee63-bfe9-fcdf79afd6b9/data/data1/current/BP-1583467146-172.17.0.2-1732422473010/current, will proceed with Du for space computation calculation, 
2024-11-24T04:27:54,765 WARN  [Thread-95 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/03752be3-12ba-e59c-4567-8f2d22dd32ed/cluster_6735412e-b74b-ee63-bfe9-fcdf79afd6b9/data/data3/current/BP-1583467146-172.17.0.2-1732422473010/current, will proceed with Du for space computation calculation, 
2024-11-24T04:27:54,765 WARN  [Thread-96 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/03752be3-12ba-e59c-4567-8f2d22dd32ed/cluster_6735412e-b74b-ee63-bfe9-fcdf79afd6b9/data/data4/current/BP-1583467146-172.17.0.2-1732422473010/current, will proceed with Du for space computation calculation, 
2024-11-24T04:27:54,765 WARN  [Thread-98 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/03752be3-12ba-e59c-4567-8f2d22dd32ed/cluster_6735412e-b74b-ee63-bfe9-fcdf79afd6b9/data/data2/current/BP-1583467146-172.17.0.2-1732422473010/current, will proceed with Du for space computation calculation, 
2024-11-24T04:27:54,830 WARN  [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-24T04:27:54,831 WARN  [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-24T04:27:54,909 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8adb72f34b66b8c0 with lease ID 0x84588e617a0545ac: Processing first storage report for DS-f52c0a13-f8de-439e-9980-ba843e0b1c2f from datanode DatanodeRegistration(127.0.0.1:34465, datanodeUuid=b765d968-3d03-437c-b7ca-4ee12294f80f, infoPort=38793, infoSecurePort=0, ipcPort=43299, storageInfo=lv=-57;cid=testClusterID;nsid=1511428982;c=1732422473010)
2024-11-24T04:27:54,912 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8adb72f34b66b8c0 with lease ID 0x84588e617a0545ac: from storage DS-f52c0a13-f8de-439e-9980-ba843e0b1c2f node DatanodeRegistration(127.0.0.1:34465, datanodeUuid=b765d968-3d03-437c-b7ca-4ee12294f80f, infoPort=38793, infoSecurePort=0, ipcPort=43299, storageInfo=lv=-57;cid=testClusterID;nsid=1511428982;c=1732422473010), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0
2024-11-24T04:27:54,912 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8062c103bc15116 with lease ID 0x84588e617a0545ad: Processing first storage report for DS-576af977-24e7-407d-9d91-3882eae2681d from datanode DatanodeRegistration(127.0.0.1:34329, datanodeUuid=3d793753-93ca-4950-92cf-102a55538428, infoPort=42911, infoSecurePort=0, ipcPort=36449, storageInfo=lv=-57;cid=testClusterID;nsid=1511428982;c=1732422473010)
2024-11-24T04:27:54,913 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8062c103bc15116 with lease ID 0x84588e617a0545ad: from storage DS-576af977-24e7-407d-9d91-3882eae2681d node DatanodeRegistration(127.0.0.1:34329, datanodeUuid=3d793753-93ca-4950-92cf-102a55538428, infoPort=42911, infoSecurePort=0, ipcPort=36449, storageInfo=lv=-57;cid=testClusterID;nsid=1511428982;c=1732422473010), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T04:27:54,913 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8adb72f34b66b8c0 with lease ID 0x84588e617a0545ac: Processing first storage report for DS-1de97fd2-f2fd-4e01-afae-73f1a39e76ad from datanode DatanodeRegistration(127.0.0.1:34465, datanodeUuid=b765d968-3d03-437c-b7ca-4ee12294f80f, infoPort=38793, infoSecurePort=0, ipcPort=43299, storageInfo=lv=-57;cid=testClusterID;nsid=1511428982;c=1732422473010)
2024-11-24T04:27:54,913 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8adb72f34b66b8c0 with lease ID 0x84588e617a0545ac: from storage DS-1de97fd2-f2fd-4e01-afae-73f1a39e76ad node DatanodeRegistration(127.0.0.1:34465, datanodeUuid=b765d968-3d03-437c-b7ca-4ee12294f80f, infoPort=38793, infoSecurePort=0, ipcPort=43299, storageInfo=lv=-57;cid=testClusterID;nsid=1511428982;c=1732422473010), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T04:27:54,913 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8062c103bc15116 with lease ID 0x84588e617a0545ad: Processing first storage report for DS-7091b80c-ab58-4e86-ab76-9efe8200fc30 from datanode DatanodeRegistration(127.0.0.1:34329, datanodeUuid=3d793753-93ca-4950-92cf-102a55538428, infoPort=42911, infoSecurePort=0, ipcPort=36449, storageInfo=lv=-57;cid=testClusterID;nsid=1511428982;c=1732422473010)
2024-11-24T04:27:54,914 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8062c103bc15116 with lease ID 0x84588e617a0545ad: from storage DS-7091b80c-ab58-4e86-ab76-9efe8200fc30 node DatanodeRegistration(127.0.0.1:34329, datanodeUuid=3d793753-93ca-4950-92cf-102a55538428, infoPort=42911, infoSecurePort=0, ipcPort=36449, storageInfo=lv=-57;cid=testClusterID;nsid=1511428982;c=1732422473010), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-24T04:27:54,970 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/03752be3-12ba-e59c-4567-8f2d22dd32ed
2024-11-24T04:27:55,043 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/03752be3-12ba-e59c-4567-8f2d22dd32ed/cluster_6735412e-b74b-ee63-bfe9-fcdf79afd6b9/zookeeper_0, clientPort=57793, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/03752be3-12ba-e59c-4567-8f2d22dd32ed/cluster_6735412e-b74b-ee63-bfe9-fcdf79afd6b9/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/03752be3-12ba-e59c-4567-8f2d22dd32ed/cluster_6735412e-b74b-ee63-bfe9-fcdf79afd6b9/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-24T04:27:55,053 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=57793
2024-11-24T04:27:55,063 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T04:27:55,066 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T04:27:55,308 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34465 is added to blk_1073741825_1001 (size=7)
2024-11-24T04:27:55,309 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741825_1001 (size=7)
2024-11-24T04:27:55,716 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165 with version=8
2024-11-24T04:27:55,716 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/hbase-staging
2024-11-24T04:27:55,807 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16
2024-11-24T04:27:56,057 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/fadb2fe0ba32:0 server-side Connection retries=45
2024-11-24T04:27:56,068 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-24T04:27:56,069 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-24T04:27:56,073 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-24T04:27:56,073 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-24T04:27:56,074 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-24T04:27:56,210 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-24T04:27:56,270 INFO  [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl
2024-11-24T04:27:56,278 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout
2024-11-24T04:27:56,282 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-24T04:27:56,309 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 19546 (auto-detected)
2024-11-24T04:27:56,310 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected)
2024-11-24T04:27:56,329 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43529
2024-11-24T04:27:56,350 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43529 connecting to ZooKeeper ensemble=127.0.0.1:57793
2024-11-24T04:27:56,382 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:435290x0, quorum=127.0.0.1:57793, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-24T04:27:56,384 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43529-0x101483b07a20000 connected
2024-11-24T04:27:56,415 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T04:27:56,417 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T04:27:56,428 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43529-0x101483b07a20000, quorum=127.0.0.1:57793, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-24T04:27:56,433 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165, hbase.cluster.distributed=false
2024-11-24T04:27:56,463 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43529-0x101483b07a20000, quorum=127.0.0.1:57793, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-24T04:27:56,470 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43529
2024-11-24T04:27:56,472 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43529
2024-11-24T04:27:56,472 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43529
2024-11-24T04:27:56,473 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43529
2024-11-24T04:27:56,473 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43529
2024-11-24T04:27:56,599 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/fadb2fe0ba32:0 server-side Connection retries=45
2024-11-24T04:27:56,601 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-24T04:27:56,601 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-24T04:27:56,602 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-24T04:27:56,602 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-24T04:27:56,602 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-24T04:27:56,605 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-24T04:27:56,607 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-24T04:27:56,608 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41193
2024-11-24T04:27:56,610 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41193 connecting to ZooKeeper ensemble=127.0.0.1:57793
2024-11-24T04:27:56,611 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T04:27:56,615 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T04:27:56,623 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:411930x0, quorum=127.0.0.1:57793, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-24T04:27:56,624 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:411930x0, quorum=127.0.0.1:57793, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-24T04:27:56,624 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41193-0x101483b07a20001 connected
2024-11-24T04:27:56,628 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-24T04:27:56,638 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-24T04:27:56,640 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41193-0x101483b07a20001, quorum=127.0.0.1:57793, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-24T04:27:56,646 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41193-0x101483b07a20001, quorum=127.0.0.1:57793, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-24T04:27:56,647 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41193
2024-11-24T04:27:56,648 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41193
2024-11-24T04:27:56,648 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41193
2024-11-24T04:27:56,650 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41193
2024-11-24T04:27:56,652 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41193
2024-11-24T04:27:56,670 DEBUG [M:0;fadb2fe0ba32:43529 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;fadb2fe0ba32:43529
2024-11-24T04:27:56,671 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/fadb2fe0ba32,43529,1732422475859
2024-11-24T04:27:56,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41193-0x101483b07a20001, quorum=127.0.0.1:57793, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-24T04:27:56,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43529-0x101483b07a20000, quorum=127.0.0.1:57793, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-24T04:27:56,680 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43529-0x101483b07a20000, quorum=127.0.0.1:57793, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/fadb2fe0ba32,43529,1732422475859
2024-11-24T04:27:56,702 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41193-0x101483b07a20001, quorum=127.0.0.1:57793, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-24T04:27:56,702 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43529-0x101483b07a20000, quorum=127.0.0.1:57793, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:27:56,702 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41193-0x101483b07a20001, quorum=127.0.0.1:57793, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:27:56,703 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43529-0x101483b07a20000, quorum=127.0.0.1:57793, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-24T04:27:56,704 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/fadb2fe0ba32,43529,1732422475859 from backup master directory
2024-11-24T04:27:56,707 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43529-0x101483b07a20000, quorum=127.0.0.1:57793, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/fadb2fe0ba32,43529,1732422475859
2024-11-24T04:27:56,707 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41193-0x101483b07a20001, quorum=127.0.0.1:57793, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-24T04:27:56,708 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43529-0x101483b07a20000, quorum=127.0.0.1:57793, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-24T04:27:56,708 WARN  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-24T04:27:56,708 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=fadb2fe0ba32,43529,1732422475859
2024-11-24T04:27:56,711 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0
2024-11-24T04:27:56,712 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0
2024-11-24T04:27:56,771 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/hbase.id] with ID: 54795bb2-19d9-42f4-b82f-bac4cef64c7f
2024-11-24T04:27:56,771 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/.tmp/hbase.id
2024-11-24T04:27:56,782 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741826_1002 (size=42)
2024-11-24T04:27:56,783 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34465 is added to blk_1073741826_1002 (size=42)
2024-11-24T04:27:56,784 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/.tmp/hbase.id]:[hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/hbase.id]
2024-11-24T04:27:56,826 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T04:27:56,831 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-24T04:27:56,850 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms.
2024-11-24T04:27:56,854 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41193-0x101483b07a20001, quorum=127.0.0.1:57793, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:27:56,854 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43529-0x101483b07a20000, quorum=127.0.0.1:57793, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:27:56,870 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34465 is added to blk_1073741827_1003 (size=196)
2024-11-24T04:27:56,870 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741827_1003 (size=196)
2024-11-24T04:27:56,886 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-24T04:27:56,889 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-24T04:27:56,897 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-24T04:27:56,935 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34465 is added to blk_1073741828_1004 (size=1189)
2024-11-24T04:27:56,935 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741828_1004 (size=1189)
2024-11-24T04:27:56,953 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/MasterData/data/master/store
2024-11-24T04:27:56,973 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741829_1005 (size=34)
2024-11-24T04:27:56,973 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34465 is added to blk_1073741829_1005 (size=34)
2024-11-24T04:27:56,978 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure.
2024-11-24T04:27:56,981 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T04:27:56,982 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-24T04:27:56,982 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T04:27:56,982 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T04:27:56,984 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-24T04:27:56,984 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T04:27:56,984 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T04:27:56,985 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732422476982Disabling compacts and flushes for region at 1732422476982Disabling writes for close at 1732422476984 (+2 ms)Writing region close event to WAL at 1732422476984Closed at 1732422476984
2024-11-24T04:27:56,988 WARN  [master/fadb2fe0ba32:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/MasterData/data/master/store/.initializing
2024-11-24T04:27:56,988 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/MasterData/WALs/fadb2fe0ba32,43529,1732422475859
2024-11-24T04:27:57,011 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fadb2fe0ba32%2C43529%2C1732422475859, suffix=, logDir=hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/MasterData/WALs/fadb2fe0ba32,43529,1732422475859, archiveDir=hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/MasterData/oldWALs, maxLogs=10
2024-11-24T04:27:57,023 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor fadb2fe0ba32%2C43529%2C1732422475859.1732422477017
2024-11-24T04:27:57,047 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/MasterData/WALs/fadb2fe0ba32,43529,1732422475859/fadb2fe0ba32%2C43529%2C1732422475859.1732422477017
2024-11-24T04:27:57,059 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38793:38793),(127.0.0.1/127.0.0.1:42911:42911)]
2024-11-24T04:27:57,061 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-24T04:27:57,061 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T04:27:57,065 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T04:27:57,066 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T04:27:57,104 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-24T04:27:57,131 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-24T04:27:57,135 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:27:57,137 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T04:27:57,138 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-24T04:27:57,141 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-24T04:27:57,141 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:27:57,142 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-24T04:27:57,142 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-24T04:27:57,145 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-24T04:27:57,145 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:27:57,146 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-24T04:27:57,146 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-24T04:27:57,149 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-24T04:27:57,149 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:27:57,150 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-24T04:27:57,150 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T04:27:57,154 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-24T04:27:57,155 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-24T04:27:57,161 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T04:27:57,162 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T04:27:57,165 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-24T04:27:57,169 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T04:27:57,173 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-24T04:27:57,174 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=773452, jitterRate=-0.01650586724281311}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-24T04:27:57,180 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1732422477078Initializing all the Stores at 1732422477080 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732422477081 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422477081Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422477082 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422477082Cleaning up temporary data from old regions at 1732422477162 (+80 ms)Region opened successfully at 1732422477180 (+18 ms)
2024-11-24T04:27:57,182 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-24T04:27:57,218 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@791e899c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fadb2fe0ba32/172.17.0.2:0
2024-11-24T04:27:57,251 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-24T04:27:57,263 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-24T04:27:57,263 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-24T04:27:57,266 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-24T04:27:57,268 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec
2024-11-24T04:27:57,272 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec
2024-11-24T04:27:57,273 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-24T04:27:57,297 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-24T04:27:57,306 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43529-0x101483b07a20000, quorum=127.0.0.1:57793, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-24T04:27:57,307 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-24T04:27:57,310 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-24T04:27:57,311 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43529-0x101483b07a20000, quorum=127.0.0.1:57793, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-24T04:27:57,319 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-24T04:27:57,322 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-24T04:27:57,325 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43529-0x101483b07a20000, quorum=127.0.0.1:57793, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-24T04:27:57,326 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-24T04:27:57,328 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43529-0x101483b07a20000, quorum=127.0.0.1:57793, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-24T04:27:57,329 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-24T04:27:57,346 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43529-0x101483b07a20000, quorum=127.0.0.1:57793, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-24T04:27:57,348 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-24T04:27:57,353 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41193-0x101483b07a20001, quorum=127.0.0.1:57793, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-24T04:27:57,353 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43529-0x101483b07a20000, quorum=127.0.0.1:57793, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-24T04:27:57,353 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41193-0x101483b07a20001, quorum=127.0.0.1:57793, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:27:57,353 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43529-0x101483b07a20000, quorum=127.0.0.1:57793, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:27:57,357 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=fadb2fe0ba32,43529,1732422475859, sessionid=0x101483b07a20000, setting cluster-up flag (Was=false)
2024-11-24T04:27:57,369 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43529-0x101483b07a20000, quorum=127.0.0.1:57793, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:27:57,369 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41193-0x101483b07a20001, quorum=127.0.0.1:57793, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:27:57,377 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-24T04:27:57,379 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=fadb2fe0ba32,43529,1732422475859
2024-11-24T04:27:57,384 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41193-0x101483b07a20001, quorum=127.0.0.1:57793, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:27:57,384 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43529-0x101483b07a20000, quorum=127.0.0.1:57793, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:27:57,392 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-24T04:27:57,394 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=fadb2fe0ba32,43529,1732422475859
2024-11-24T04:27:57,402 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-24T04:27:57,457 INFO  [RS:0;fadb2fe0ba32:41193 {}] regionserver.HRegionServer(746): ClusterId : 54795bb2-19d9-42f4-b82f-bac4cef64c7f
2024-11-24T04:27:57,459 DEBUG [RS:0;fadb2fe0ba32:41193 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-24T04:27:57,465 DEBUG [RS:0;fadb2fe0ba32:41193 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-24T04:27:57,465 DEBUG [RS:0;fadb2fe0ba32:41193 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-24T04:27:57,470 DEBUG [RS:0;fadb2fe0ba32:41193 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-24T04:27:57,471 DEBUG [RS:0;fadb2fe0ba32:41193 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ecb6ea5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fadb2fe0ba32/172.17.0.2:0
2024-11-24T04:27:57,476 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-24T04:27:57,485 DEBUG [RS:0;fadb2fe0ba32:41193 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;fadb2fe0ba32:41193
2024-11-24T04:27:57,485 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-24T04:27:57,488 INFO  [RS:0;fadb2fe0ba32:41193 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-24T04:27:57,488 INFO  [RS:0;fadb2fe0ba32:41193 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-24T04:27:57,488 DEBUG [RS:0;fadb2fe0ba32:41193 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-24T04:27:57,490 INFO  [RS:0;fadb2fe0ba32:41193 {}] regionserver.HRegionServer(2659): reportForDuty to master=fadb2fe0ba32,43529,1732422475859 with port=41193, startcode=1732422476559
2024-11-24T04:27:57,491 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-24T04:27:57,497 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: fadb2fe0ba32,43529,1732422475859
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-24T04:27:57,502 DEBUG [RS:0;fadb2fe0ba32:41193 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-24T04:27:57,504 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/fadb2fe0ba32:0, corePoolSize=5, maxPoolSize=5
2024-11-24T04:27:57,505 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/fadb2fe0ba32:0, corePoolSize=5, maxPoolSize=5
2024-11-24T04:27:57,505 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/fadb2fe0ba32:0, corePoolSize=5, maxPoolSize=5
2024-11-24T04:27:57,505 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/fadb2fe0ba32:0, corePoolSize=5, maxPoolSize=5
2024-11-24T04:27:57,505 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/fadb2fe0ba32:0, corePoolSize=10, maxPoolSize=10
2024-11-24T04:27:57,505 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:27:57,505 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/fadb2fe0ba32:0, corePoolSize=2, maxPoolSize=2
2024-11-24T04:27:57,506 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:27:57,508 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732422507508
2024-11-24T04:27:57,510 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-24T04:27:57,511 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-24T04:27:57,512 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-24T04:27:57,513 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-24T04:27:57,514 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-24T04:27:57,515 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-24T04:27:57,515 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-24T04:27:57,515 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-24T04:27:57,516 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-24T04:27:57,518 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-24T04:27:57,519 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-24T04:27:57,520 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:27:57,520 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-24T04:27:57,520 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-24T04:27:57,522 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-24T04:27:57,522 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-24T04:27:57,526 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/fadb2fe0ba32:0:becomeActiveMaster-HFileCleaner.large.0-1732422477523,5,FailOnTimeoutGroup]
2024-11-24T04:27:57,526 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/fadb2fe0ba32:0:becomeActiveMaster-HFileCleaner.small.0-1732422477526,5,FailOnTimeoutGroup]
2024-11-24T04:27:57,527 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-24T04:27:57,527 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-24T04:27:57,528 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-24T04:27:57,529 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-24T04:27:57,540 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741831_1007 (size=1321)
2024-11-24T04:27:57,541 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34465 is added to blk_1073741831_1007 (size=1321)
2024-11-24T04:27:57,577 INFO  [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54793, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService
2024-11-24T04:27:57,585 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43529 {}] master.ServerManager(363): Checking decommissioned status of RegionServer fadb2fe0ba32,41193,1732422476559
2024-11-24T04:27:57,588 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43529 {}] master.ServerManager(517): Registering regionserver=fadb2fe0ba32,41193,1732422476559
2024-11-24T04:27:57,603 DEBUG [RS:0;fadb2fe0ba32:41193 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165
2024-11-24T04:27:57,604 DEBUG [RS:0;fadb2fe0ba32:41193 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35431
2024-11-24T04:27:57,604 DEBUG [RS:0;fadb2fe0ba32:41193 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-24T04:27:57,608 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43529-0x101483b07a20000, quorum=127.0.0.1:57793, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-24T04:27:57,608 DEBUG [RS:0;fadb2fe0ba32:41193 {}] zookeeper.ZKUtil(111): regionserver:41193-0x101483b07a20001, quorum=127.0.0.1:57793, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/fadb2fe0ba32,41193,1732422476559
2024-11-24T04:27:57,608 WARN  [RS:0;fadb2fe0ba32:41193 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-24T04:27:57,609 INFO  [RS:0;fadb2fe0ba32:41193 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-24T04:27:57,609 DEBUG [RS:0;fadb2fe0ba32:41193 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/WALs/fadb2fe0ba32,41193,1732422476559
2024-11-24T04:27:57,610 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [fadb2fe0ba32,41193,1732422476559]
2024-11-24T04:27:57,634 INFO  [RS:0;fadb2fe0ba32:41193 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-24T04:27:57,648 INFO  [RS:0;fadb2fe0ba32:41193 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-24T04:27:57,652 INFO  [RS:0;fadb2fe0ba32:41193 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-24T04:27:57,653 INFO  [RS:0;fadb2fe0ba32:41193 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-24T04:27:57,654 INFO  [RS:0;fadb2fe0ba32:41193 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-24T04:27:57,660 INFO  [RS:0;fadb2fe0ba32:41193 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-24T04:27:57,661 INFO  [RS:0;fadb2fe0ba32:41193 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-24T04:27:57,661 DEBUG [RS:0;fadb2fe0ba32:41193 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:27:57,661 DEBUG [RS:0;fadb2fe0ba32:41193 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:27:57,662 DEBUG [RS:0;fadb2fe0ba32:41193 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:27:57,662 DEBUG [RS:0;fadb2fe0ba32:41193 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:27:57,662 DEBUG [RS:0;fadb2fe0ba32:41193 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:27:57,662 DEBUG [RS:0;fadb2fe0ba32:41193 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/fadb2fe0ba32:0, corePoolSize=2, maxPoolSize=2
2024-11-24T04:27:57,662 DEBUG [RS:0;fadb2fe0ba32:41193 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:27:57,662 DEBUG [RS:0;fadb2fe0ba32:41193 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:27:57,662 DEBUG [RS:0;fadb2fe0ba32:41193 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:27:57,663 DEBUG [RS:0;fadb2fe0ba32:41193 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:27:57,663 DEBUG [RS:0;fadb2fe0ba32:41193 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:27:57,663 DEBUG [RS:0;fadb2fe0ba32:41193 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:27:57,663 DEBUG [RS:0;fadb2fe0ba32:41193 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/fadb2fe0ba32:0, corePoolSize=3, maxPoolSize=3
2024-11-24T04:27:57,663 DEBUG [RS:0;fadb2fe0ba32:41193 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/fadb2fe0ba32:0, corePoolSize=3, maxPoolSize=3
2024-11-24T04:27:57,664 INFO  [RS:0;fadb2fe0ba32:41193 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-24T04:27:57,664 INFO  [RS:0;fadb2fe0ba32:41193 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-24T04:27:57,664 INFO  [RS:0;fadb2fe0ba32:41193 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-24T04:27:57,664 INFO  [RS:0;fadb2fe0ba32:41193 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-24T04:27:57,664 INFO  [RS:0;fadb2fe0ba32:41193 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-24T04:27:57,665 INFO  [RS:0;fadb2fe0ba32:41193 {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,41193,1732422476559-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-24T04:27:57,684 INFO  [RS:0;fadb2fe0ba32:41193 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-24T04:27:57,686 INFO  [RS:0;fadb2fe0ba32:41193 {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,41193,1732422476559-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-24T04:27:57,686 INFO  [RS:0;fadb2fe0ba32:41193 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T04:27:57,687 INFO  [RS:0;fadb2fe0ba32:41193 {}] regionserver.Replication(171): fadb2fe0ba32,41193,1732422476559 started
2024-11-24T04:27:57,708 INFO  [RS:0;fadb2fe0ba32:41193 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T04:27:57,708 INFO  [RS:0;fadb2fe0ba32:41193 {}] regionserver.HRegionServer(1482): Serving as fadb2fe0ba32,41193,1732422476559, RpcServer on fadb2fe0ba32/172.17.0.2:41193, sessionid=0x101483b07a20001
2024-11-24T04:27:57,709 DEBUG [RS:0;fadb2fe0ba32:41193 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-24T04:27:57,709 DEBUG [RS:0;fadb2fe0ba32:41193 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager fadb2fe0ba32,41193,1732422476559
2024-11-24T04:27:57,710 DEBUG [RS:0;fadb2fe0ba32:41193 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fadb2fe0ba32,41193,1732422476559'
2024-11-24T04:27:57,710 DEBUG [RS:0;fadb2fe0ba32:41193 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-24T04:27:57,711 DEBUG [RS:0;fadb2fe0ba32:41193 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-24T04:27:57,712 DEBUG [RS:0;fadb2fe0ba32:41193 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-24T04:27:57,712 DEBUG [RS:0;fadb2fe0ba32:41193 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-24T04:27:57,712 DEBUG [RS:0;fadb2fe0ba32:41193 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager fadb2fe0ba32,41193,1732422476559
2024-11-24T04:27:57,712 DEBUG [RS:0;fadb2fe0ba32:41193 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fadb2fe0ba32,41193,1732422476559'
2024-11-24T04:27:57,712 DEBUG [RS:0;fadb2fe0ba32:41193 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-24T04:27:57,713 DEBUG [RS:0;fadb2fe0ba32:41193 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-24T04:27:57,714 DEBUG [RS:0;fadb2fe0ba32:41193 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-24T04:27:57,714 INFO  [RS:0;fadb2fe0ba32:41193 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-24T04:27:57,714 INFO  [RS:0;fadb2fe0ba32:41193 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-24T04:27:57,822 INFO  [RS:0;fadb2fe0ba32:41193 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fadb2fe0ba32%2C41193%2C1732422476559, suffix=, logDir=hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/WALs/fadb2fe0ba32,41193,1732422476559, archiveDir=hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/oldWALs, maxLogs=32
2024-11-24T04:27:57,825 INFO  [RS:0;fadb2fe0ba32:41193 {}] monitor.StreamSlowMonitor(122): New stream slow monitor fadb2fe0ba32%2C41193%2C1732422476559.1732422477825
2024-11-24T04:27:57,834 INFO  [RS:0;fadb2fe0ba32:41193 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/WALs/fadb2fe0ba32,41193,1732422476559/fadb2fe0ba32%2C41193%2C1732422476559.1732422477825
2024-11-24T04:27:57,835 DEBUG [RS:0;fadb2fe0ba32:41193 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38793:38793),(127.0.0.1/127.0.0.1:42911:42911)]
2024-11-24T04:27:57,940 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-24T04:27:57,941 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165
2024-11-24T04:27:57,950 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34465 is added to blk_1073741833_1009 (size=32)
2024-11-24T04:27:57,950 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741833_1009 (size=32)
2024-11-24T04:27:57,952 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T04:27:57,954 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-24T04:27:57,957 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-24T04:27:57,957 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:27:57,958 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T04:27:57,958 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-24T04:27:57,960 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-24T04:27:57,960 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:27:57,961 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T04:27:57,961 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-24T04:27:57,963 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-24T04:27:57,963 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:27:57,964 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T04:27:57,964 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-24T04:27:57,966 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-24T04:27:57,967 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:27:57,967 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T04:27:57,968 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-24T04:27:57,969 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/hbase/meta/1588230740
2024-11-24T04:27:57,970 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/hbase/meta/1588230740
2024-11-24T04:27:57,972 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-24T04:27:57,972 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-24T04:27:57,973 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-24T04:27:57,976 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-24T04:27:57,979 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-24T04:27:57,980 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=871370, jitterRate=0.10800525546073914}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-24T04:27:57,983 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1732422477952Initializing all the Stores at 1732422477954 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732422477954Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732422477954Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422477954Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732422477954Cleaning up temporary data from old regions at 1732422477972 (+18 ms)Region opened successfully at 1732422477982 (+10 ms)
2024-11-24T04:27:57,983 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-24T04:27:57,983 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-24T04:27:57,983 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-24T04:27:57,983 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-24T04:27:57,983 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-24T04:27:57,984 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-24T04:27:57,984 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732422477983Disabling compacts and flushes for region at 1732422477983Disabling writes for close at 1732422477983Writing region close event to WAL at 1732422477984 (+1 ms)Closed at 1732422477984
2024-11-24T04:27:57,987 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-24T04:27:57,988 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-24T04:27:57,994 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-24T04:27:58,002 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-24T04:27:58,005 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-24T04:27:58,158 DEBUG [fadb2fe0ba32:43529 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-24T04:27:58,171 INFO  [PEWorker-2 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=fadb2fe0ba32,41193,1732422476559
2024-11-24T04:27:58,178 INFO  [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fadb2fe0ba32,41193,1732422476559, state=OPENING
2024-11-24T04:27:58,183 DEBUG [PEWorker-2 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-24T04:27:58,185 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43529-0x101483b07a20000, quorum=127.0.0.1:57793, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:27:58,185 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41193-0x101483b07a20001, quorum=127.0.0.1:57793, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:27:58,186 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-24T04:27:58,186 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-24T04:27:58,187 DEBUG [PEWorker-2 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-24T04:27:58,189 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=fadb2fe0ba32,41193,1732422476559}]
2024-11-24T04:27:58,364 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-24T04:27:58,368 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36677, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-24T04:27:58,378 INFO  [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-24T04:27:58,379 INFO  [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-24T04:27:58,382 INFO  [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fadb2fe0ba32%2C41193%2C1732422476559.meta, suffix=.meta, logDir=hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/WALs/fadb2fe0ba32,41193,1732422476559, archiveDir=hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/oldWALs, maxLogs=32
2024-11-24T04:27:58,384 INFO  [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor fadb2fe0ba32%2C41193%2C1732422476559.meta.1732422478384.meta
2024-11-24T04:27:58,392 INFO  [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/WALs/fadb2fe0ba32,41193,1732422476559/fadb2fe0ba32%2C41193%2C1732422476559.meta.1732422478384.meta
2024-11-24T04:27:58,396 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42911:42911),(127.0.0.1/127.0.0.1:38793:38793)]
2024-11-24T04:27:58,397 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-24T04:27:58,399 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-24T04:27:58,402 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-24T04:27:58,406 INFO  [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-24T04:27:58,411 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-24T04:27:58,411 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T04:27:58,411 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-24T04:27:58,411 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-24T04:27:58,414 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-24T04:27:58,416 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-24T04:27:58,416 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:27:58,417 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T04:27:58,417 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-24T04:27:58,418 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-24T04:27:58,418 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:27:58,419 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T04:27:58,419 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-24T04:27:58,420 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-24T04:27:58,420 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:27:58,421 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T04:27:58,421 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-24T04:27:58,423 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-24T04:27:58,423 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:27:58,423 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T04:27:58,424 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-24T04:27:58,425 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/hbase/meta/1588230740
2024-11-24T04:27:58,427 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/hbase/meta/1588230740
2024-11-24T04:27:58,430 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-24T04:27:58,430 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-24T04:27:58,431 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-24T04:27:58,433 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-24T04:27:58,435 INFO  [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=874709, jitterRate=0.11225026845932007}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-24T04:27:58,435 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-24T04:27:58,437 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1732422478412Writing region info on filesystem at 1732422478412Initializing all the Stores at 1732422478414 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732422478414Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732422478414Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422478414Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732422478414Cleaning up temporary data from old regions at 1732422478430 (+16 ms)Running coprocessor post-open hooks at 1732422478435 (+5 ms)Region opened successfully at 1732422478437 (+2 ms)
2024-11-24T04:27:58,444 INFO  [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732422478354
2024-11-24T04:27:58,456 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-24T04:27:58,457 INFO  [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-24T04:27:58,459 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=fadb2fe0ba32,41193,1732422476559
2024-11-24T04:27:58,461 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fadb2fe0ba32,41193,1732422476559, state=OPEN
2024-11-24T04:27:58,467 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41193-0x101483b07a20001, quorum=127.0.0.1:57793, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-24T04:27:58,467 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43529-0x101483b07a20000, quorum=127.0.0.1:57793, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-24T04:27:58,467 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-24T04:27:58,467 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-24T04:27:58,468 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=fadb2fe0ba32,41193,1732422476559
2024-11-24T04:27:58,474 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-24T04:27:58,474 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fadb2fe0ba32,41193,1732422476559 in 280 msec
2024-11-24T04:27:58,481 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-24T04:27:58,481 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 482 msec
2024-11-24T04:27:58,483 DEBUG [PEWorker-4 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-24T04:27:58,483 INFO  [PEWorker-4 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-24T04:27:58,503 DEBUG [PEWorker-4 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-24T04:27:58,504 DEBUG [PEWorker-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fadb2fe0ba32,41193,1732422476559, seqNum=-1]
2024-11-24T04:27:58,526 DEBUG [PEWorker-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-24T04:27:58,529 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51081, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-24T04:27:58,549 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1170 sec
2024-11-24T04:27:58,549 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732422478549, completionTime=-1
2024-11-24T04:27:58,552 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-24T04:27:58,552 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-24T04:27:58,581 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-24T04:27:58,581 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732422538581
2024-11-24T04:27:58,581 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732422598581
2024-11-24T04:27:58,581 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 28 msec
2024-11-24T04:27:58,584 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,43529,1732422475859-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-24T04:27:58,584 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,43529,1732422475859-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T04:27:58,585 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,43529,1732422475859-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T04:27:58,586 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-fadb2fe0ba32:43529, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T04:27:58,587 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-24T04:27:58,587 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-24T04:27:58,594 DEBUG [master/fadb2fe0ba32:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-24T04:27:58,615 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.906sec
2024-11-24T04:27:58,616 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-24T04:27:58,617 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-24T04:27:58,618 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-24T04:27:58,619 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-24T04:27:58,619 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-24T04:27:58,620 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,43529,1732422475859-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-24T04:27:58,620 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,43529,1732422475859-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-24T04:27:58,629 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-24T04:27:58,630 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-24T04:27:58,631 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,43529,1732422475859-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T04:27:58,667 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4731d90b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-24T04:27:58,669 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false
2024-11-24T04:27:58,669 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512
2024-11-24T04:27:58,673 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fadb2fe0ba32,43529,-1 for getting cluster id
2024-11-24T04:27:58,675 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-24T04:27:58,684 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '54795bb2-19d9-42f4-b82f-bac4cef64c7f'
2024-11-24T04:27:58,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-24T04:27:58,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "54795bb2-19d9-42f4-b82f-bac4cef64c7f"

2024-11-24T04:27:58,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4974ec01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-24T04:27:58,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fadb2fe0ba32,43529,-1]
2024-11-24T04:27:58,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-24T04:27:58,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T04:27:58,695 INFO  [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49782, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-24T04:27:58,698 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2eb3700d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-24T04:27:58,699 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-24T04:27:58,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fadb2fe0ba32,41193,1732422476559, seqNum=-1]
2024-11-24T04:27:58,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-24T04:27:58,709 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48122, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-24T04:27:58,730 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=fadb2fe0ba32,43529,1732422475859
2024-11-24T04:27:58,731 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T04:27:58,739 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-24T04:27:58,742 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-24T04:27:58,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is fadb2fe0ba32,43529,1732422475859
2024-11-24T04:27:58,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3356f785
2024-11-24T04:27:58,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-24T04:27:58,755 INFO  [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49790, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-24T04:27:58,757 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43529 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-24T04:27:58,757 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43529 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-24T04:27:58,760 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43529 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testSlowSyncLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-24T04:27:58,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43529 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling
2024-11-24T04:27:58,771 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-11-24T04:27:58,773 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43529 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testSlowSyncLogRolling"
 procId is: 4
2024-11-24T04:27:58,774 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:27:58,776 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-24T04:27:58,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43529 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-24T04:27:58,829 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34465 is added to blk_1073741835_1011 (size=389)
2024-11-24T04:27:58,829 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741835_1011 (size=389)
2024-11-24T04:27:58,833 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6d7ca8a8825dc62635b3790ec1de87e1, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1732422478756.6d7ca8a8825dc62635b3790ec1de87e1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testSlowSyncLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165
2024-11-24T04:27:58,842 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741836_1012 (size=72)
2024-11-24T04:27:58,842 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34465 is added to blk_1073741836_1012 (size=72)
2024-11-24T04:27:58,844 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testSlowSyncLogRolling,,1732422478756.6d7ca8a8825dc62635b3790ec1de87e1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T04:27:58,844 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1722): Closing 6d7ca8a8825dc62635b3790ec1de87e1, disabling compactions & flushes
2024-11-24T04:27:58,844 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testSlowSyncLogRolling,,1732422478756.6d7ca8a8825dc62635b3790ec1de87e1.
2024-11-24T04:27:58,844 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testSlowSyncLogRolling,,1732422478756.6d7ca8a8825dc62635b3790ec1de87e1.
2024-11-24T04:27:58,844 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1732422478756.6d7ca8a8825dc62635b3790ec1de87e1. after waiting 0 ms
2024-11-24T04:27:58,844 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1732422478756.6d7ca8a8825dc62635b3790ec1de87e1.
2024-11-24T04:27:58,844 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testSlowSyncLogRolling,,1732422478756.6d7ca8a8825dc62635b3790ec1de87e1.
2024-11-24T04:27:58,844 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6d7ca8a8825dc62635b3790ec1de87e1:
Waiting for close lock at 1732422478844Disabling compacts and flushes for region at 1732422478844Disabling writes for close at 1732422478844Writing region close event to WAL at 1732422478844Closed at 1732422478844
2024-11-24T04:27:58,847 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ADD_TO_META
2024-11-24T04:27:58,852 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testSlowSyncLogRolling,,1732422478756.6d7ca8a8825dc62635b3790ec1de87e1.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1732422478847"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732422478847"}]},"ts":"1732422478847"}
2024-11-24T04:27:58,857 INFO  [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-24T04:27:58,859 INFO  [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-24T04:27:58,862 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732422478859"}]},"ts":"1732422478859"}
2024-11-24T04:27:58,867 INFO  [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLING in hbase:meta
2024-11-24T04:27:58,869 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=6d7ca8a8825dc62635b3790ec1de87e1, ASSIGN}]
2024-11-24T04:27:58,871 INFO  [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=6d7ca8a8825dc62635b3790ec1de87e1, ASSIGN
2024-11-24T04:27:58,873 INFO  [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=6d7ca8a8825dc62635b3790ec1de87e1, ASSIGN; state=OFFLINE, location=fadb2fe0ba32,41193,1732422476559; forceNewPlan=false, retain=false
2024-11-24T04:27:59,024 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=6d7ca8a8825dc62635b3790ec1de87e1, regionState=OPENING, regionLocation=fadb2fe0ba32,41193,1732422476559
2024-11-24T04:27:59,029 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=6d7ca8a8825dc62635b3790ec1de87e1, ASSIGN because future has completed
2024-11-24T04:27:59,030 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6d7ca8a8825dc62635b3790ec1de87e1, server=fadb2fe0ba32,41193,1732422476559}]
2024-11-24T04:27:59,192 INFO  [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testSlowSyncLogRolling,,1732422478756.6d7ca8a8825dc62635b3790ec1de87e1.
2024-11-24T04:27:59,193 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 6d7ca8a8825dc62635b3790ec1de87e1, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1732422478756.6d7ca8a8825dc62635b3790ec1de87e1.', STARTKEY => '', ENDKEY => ''}
2024-11-24T04:27:59,193 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testSlowSyncLogRolling 6d7ca8a8825dc62635b3790ec1de87e1
2024-11-24T04:27:59,193 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testSlowSyncLogRolling,,1732422478756.6d7ca8a8825dc62635b3790ec1de87e1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T04:27:59,193 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 6d7ca8a8825dc62635b3790ec1de87e1
2024-11-24T04:27:59,193 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 6d7ca8a8825dc62635b3790ec1de87e1
2024-11-24T04:27:59,196 INFO  [StoreOpener-6d7ca8a8825dc62635b3790ec1de87e1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 6d7ca8a8825dc62635b3790ec1de87e1 
2024-11-24T04:27:59,199 INFO  [StoreOpener-6d7ca8a8825dc62635b3790ec1de87e1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6d7ca8a8825dc62635b3790ec1de87e1 columnFamilyName info
2024-11-24T04:27:59,199 DEBUG [StoreOpener-6d7ca8a8825dc62635b3790ec1de87e1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:27:59,200 INFO  [StoreOpener-6d7ca8a8825dc62635b3790ec1de87e1-1 {}] regionserver.HStore(327): Store=6d7ca8a8825dc62635b3790ec1de87e1/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-24T04:27:59,201 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 6d7ca8a8825dc62635b3790ec1de87e1
2024-11-24T04:27:59,202 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/default/TestLogRolling-testSlowSyncLogRolling/6d7ca8a8825dc62635b3790ec1de87e1
2024-11-24T04:27:59,203 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/default/TestLogRolling-testSlowSyncLogRolling/6d7ca8a8825dc62635b3790ec1de87e1
2024-11-24T04:27:59,203 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 6d7ca8a8825dc62635b3790ec1de87e1
2024-11-24T04:27:59,203 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 6d7ca8a8825dc62635b3790ec1de87e1
2024-11-24T04:27:59,206 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 6d7ca8a8825dc62635b3790ec1de87e1
2024-11-24T04:27:59,210 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/default/TestLogRolling-testSlowSyncLogRolling/6d7ca8a8825dc62635b3790ec1de87e1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-24T04:27:59,211 INFO  [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 6d7ca8a8825dc62635b3790ec1de87e1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=746778, jitterRate=-0.0504235178232193}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-24T04:27:59,211 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6d7ca8a8825dc62635b3790ec1de87e1
2024-11-24T04:27:59,212 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 6d7ca8a8825dc62635b3790ec1de87e1:
Running coprocessor pre-open hook at 1732422479194Writing region info on filesystem at 1732422479194Initializing all the Stores at 1732422479195 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422479195Cleaning up temporary data from old regions at 1732422479204 (+9 ms)Running coprocessor post-open hooks at 1732422479211 (+7 ms)Region opened successfully at 1732422479212 (+1 ms)
2024-11-24T04:27:59,214 INFO  [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testSlowSyncLogRolling,,1732422478756.6d7ca8a8825dc62635b3790ec1de87e1., pid=6, masterSystemTime=1732422479185
2024-11-24T04:27:59,218 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testSlowSyncLogRolling,,1732422478756.6d7ca8a8825dc62635b3790ec1de87e1.
2024-11-24T04:27:59,219 INFO  [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testSlowSyncLogRolling,,1732422478756.6d7ca8a8825dc62635b3790ec1de87e1.
2024-11-24T04:27:59,220 INFO  [PEWorker-2 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=6d7ca8a8825dc62635b3790ec1de87e1, regionState=OPEN, openSeqNum=2, regionLocation=fadb2fe0ba32,41193,1732422476559
2024-11-24T04:27:59,224 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6d7ca8a8825dc62635b3790ec1de87e1, server=fadb2fe0ba32,41193,1732422476559 because future has completed
2024-11-24T04:27:59,231 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-24T04:27:59,231 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 6d7ca8a8825dc62635b3790ec1de87e1, server=fadb2fe0ba32,41193,1732422476559 in 196 msec
2024-11-24T04:27:59,235 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-24T04:27:59,235 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=6d7ca8a8825dc62635b3790ec1de87e1, ASSIGN in 362 msec
2024-11-24T04:27:59,237 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-24T04:27:59,237 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732422479237"}]},"ts":"1732422479237"}
2024-11-24T04:27:59,240 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLED in hbase:meta
2024-11-24T04:27:59,242 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_POST_OPERATION
2024-11-24T04:27:59,245 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling in 479 msec
2024-11-24T04:28:03,915 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties
2024-11-24T04:28:03,960 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-24T04:28:03,961 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testSlowSyncLogRolling'
2024-11-24T04:28:06,267 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-24T04:28:06,268 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-11-24T04:28:06,270 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling
2024-11-24T04:28:06,270 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling Metrics about Tables on a single HBase RegionServer
2024-11-24T04:28:06,271 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-24T04:28:06,271 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-11-24T04:28:06,271 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-24T04:28:06,271 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-11-24T04:28:08,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43529 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-24T04:28:08,844 INFO  [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testSlowSyncLogRolling completed
2024-11-24T04:28:08,847 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testSlowSyncLogRolling,, stopping at row=TestLogRolling-testSlowSyncLogRolling ,, for max=2147483647 with caching=100
2024-11-24T04:28:08,853 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testSlowSyncLogRolling
2024-11-24T04:28:08,854 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testSlowSyncLogRolling,,1732422478756.6d7ca8a8825dc62635b3790ec1de87e1.
2024-11-24T04:28:08,855 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor fadb2fe0ba32%2C41193%2C1732422476559.1732422488854
2024-11-24T04:28:08,863 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:28:08,864 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:28:08,864 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:28:08,864 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:28:08,864 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:28:08,865 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/WALs/fadb2fe0ba32,41193,1732422476559/fadb2fe0ba32%2C41193%2C1732422476559.1732422477825 with entries=1, filesize=443 B; new WAL /user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/WALs/fadb2fe0ba32,41193,1732422476559/fadb2fe0ba32%2C41193%2C1732422476559.1732422488854
2024-11-24T04:28:08,866 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38793:38793),(127.0.0.1/127.0.0.1:42911:42911)]
2024-11-24T04:28:08,866 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/WALs/fadb2fe0ba32,41193,1732422476559/fadb2fe0ba32%2C41193%2C1732422476559.1732422477825 is not closed yet, will try archiving it next time
2024-11-24T04:28:08,868 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741832_1008 (size=451)
2024-11-24T04:28:08,868 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34465 is added to blk_1073741832_1008 (size=451)
2024-11-24T04:28:08,870 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/WALs/fadb2fe0ba32,41193,1732422476559/fadb2fe0ba32%2C41193%2C1732422476559.1732422477825 to hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/oldWALs/fadb2fe0ba32%2C41193%2C1732422476559.1732422477825
2024-11-24T04:28:08,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testSlowSyncLogRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testSlowSyncLogRolling,,1732422478756.6d7ca8a8825dc62635b3790ec1de87e1., hostname=fadb2fe0ba32,41193,1732422476559, seqNum=2]
2024-11-24T04:28:20,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41193 {}] regionserver.HRegion(8855): Flush requested on 6d7ca8a8825dc62635b3790ec1de87e1
2024-11-24T04:28:20,910 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6d7ca8a8825dc62635b3790ec1de87e1 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-24T04:28:20,972 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/default/TestLogRolling-testSlowSyncLogRolling/6d7ca8a8825dc62635b3790ec1de87e1/.tmp/info/85c72f8e829f4914ab86a7061708e885 is 1080, key is row0001/info:/1732422488878/Put/seqid=0
2024-11-24T04:28:20,983 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741838_1014 (size=12509)
2024-11-24T04:28:20,984 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34465 is added to blk_1073741838_1014 (size=12509)
2024-11-24T04:28:20,985 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/default/TestLogRolling-testSlowSyncLogRolling/6d7ca8a8825dc62635b3790ec1de87e1/.tmp/info/85c72f8e829f4914ab86a7061708e885
2024-11-24T04:28:21,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/default/TestLogRolling-testSlowSyncLogRolling/6d7ca8a8825dc62635b3790ec1de87e1/.tmp/info/85c72f8e829f4914ab86a7061708e885 as hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/default/TestLogRolling-testSlowSyncLogRolling/6d7ca8a8825dc62635b3790ec1de87e1/info/85c72f8e829f4914ab86a7061708e885
2024-11-24T04:28:21,077 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/default/TestLogRolling-testSlowSyncLogRolling/6d7ca8a8825dc62635b3790ec1de87e1/info/85c72f8e829f4914ab86a7061708e885, entries=7, sequenceid=11, filesize=12.2 K
2024-11-24T04:28:21,085 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 6d7ca8a8825dc62635b3790ec1de87e1 in 174ms, sequenceid=11, compaction requested=false
2024-11-24T04:28:21,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6d7ca8a8825dc62635b3790ec1de87e1:

2024-11-24T04:28:24,967 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-24T04:28:28,919 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor fadb2fe0ba32%2C41193%2C1732422476559.1732422508919
2024-11-24T04:28:29,127 INFO  [Time-limited test {}] wal.AbstractFSWAL(1368): Slow sync cost: 205 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34465,DS-f52c0a13-f8de-439e-9980-ba843e0b1c2f,DISK], DatanodeInfoWithStorage[127.0.0.1:34329,DS-576af977-24e7-407d-9d91-3882eae2681d,DISK]]
2024-11-24T04:28:29,128 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:28:29,128 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:28:29,128 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:28:29,128 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:28:29,128 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:28:29,128 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/WALs/fadb2fe0ba32,41193,1732422476559/fadb2fe0ba32%2C41193%2C1732422476559.1732422488854 with entries=12, filesize=12.10 KB; new WAL /user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/WALs/fadb2fe0ba32,41193,1732422476559/fadb2fe0ba32%2C41193%2C1732422476559.1732422508919
2024-11-24T04:28:29,129 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38793:38793),(127.0.0.1/127.0.0.1:42911:42911)]
2024-11-24T04:28:29,129 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/WALs/fadb2fe0ba32,41193,1732422476559/fadb2fe0ba32%2C41193%2C1732422476559.1732422488854 is not closed yet, will try archiving it next time
2024-11-24T04:28:29,131 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741837_1013 (size=12399)
2024-11-24T04:28:29,131 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34465 is added to blk_1073741837_1013 (size=12399)
2024-11-24T04:28:29,333 INFO  [FSHLog-0-hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165-prefix:fadb2fe0ba32,41193,1732422476559 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34465,DS-f52c0a13-f8de-439e-9980-ba843e0b1c2f,DISK], DatanodeInfoWithStorage[127.0.0.1:34329,DS-576af977-24e7-407d-9d91-3882eae2681d,DISK]]
2024-11-24T04:28:31,537 INFO  [FSHLog-0-hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165-prefix:fadb2fe0ba32,41193,1732422476559 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34465,DS-f52c0a13-f8de-439e-9980-ba843e0b1c2f,DISK], DatanodeInfoWithStorage[127.0.0.1:34329,DS-576af977-24e7-407d-9d91-3882eae2681d,DISK]]
2024-11-24T04:28:33,741 INFO  [FSHLog-0-hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165-prefix:fadb2fe0ba32,41193,1732422476559 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34465,DS-f52c0a13-f8de-439e-9980-ba843e0b1c2f,DISK], DatanodeInfoWithStorage[127.0.0.1:34329,DS-576af977-24e7-407d-9d91-3882eae2681d,DISK]]
2024-11-24T04:28:35,945 INFO  [FSHLog-0-hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165-prefix:fadb2fe0ba32,41193,1732422476559 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34465,DS-f52c0a13-f8de-439e-9980-ba843e0b1c2f,DISK], DatanodeInfoWithStorage[127.0.0.1:34329,DS-576af977-24e7-407d-9d91-3882eae2681d,DISK]]
2024-11-24T04:28:35,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41193 {}] regionserver.HRegion(8855): Flush requested on 6d7ca8a8825dc62635b3790ec1de87e1
2024-11-24T04:28:35,945 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6d7ca8a8825dc62635b3790ec1de87e1 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-24T04:28:36,147 INFO  [FSHLog-0-hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165-prefix:fadb2fe0ba32,41193,1732422476559 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34465,DS-f52c0a13-f8de-439e-9980-ba843e0b1c2f,DISK], DatanodeInfoWithStorage[127.0.0.1:34329,DS-576af977-24e7-407d-9d91-3882eae2681d,DISK]]
2024-11-24T04:28:36,153 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/default/TestLogRolling-testSlowSyncLogRolling/6d7ca8a8825dc62635b3790ec1de87e1/.tmp/info/f8e125503e1c4e0ab9eb20ed7cde1e09 is 1080, key is row0008/info:/1732422502909/Put/seqid=0
2024-11-24T04:28:36,160 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34465 is added to blk_1073741840_1016 (size=12509)
2024-11-24T04:28:36,161 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741840_1016 (size=12509)
2024-11-24T04:28:36,162 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/default/TestLogRolling-testSlowSyncLogRolling/6d7ca8a8825dc62635b3790ec1de87e1/.tmp/info/f8e125503e1c4e0ab9eb20ed7cde1e09
2024-11-24T04:28:36,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/default/TestLogRolling-testSlowSyncLogRolling/6d7ca8a8825dc62635b3790ec1de87e1/.tmp/info/f8e125503e1c4e0ab9eb20ed7cde1e09 as hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/default/TestLogRolling-testSlowSyncLogRolling/6d7ca8a8825dc62635b3790ec1de87e1/info/f8e125503e1c4e0ab9eb20ed7cde1e09
2024-11-24T04:28:36,182 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/default/TestLogRolling-testSlowSyncLogRolling/6d7ca8a8825dc62635b3790ec1de87e1/info/f8e125503e1c4e0ab9eb20ed7cde1e09, entries=7, sequenceid=21, filesize=12.2 K
2024-11-24T04:28:36,384 INFO  [FSHLog-0-hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165-prefix:fadb2fe0ba32,41193,1732422476559 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34465,DS-f52c0a13-f8de-439e-9980-ba843e0b1c2f,DISK], DatanodeInfoWithStorage[127.0.0.1:34329,DS-576af977-24e7-407d-9d91-3882eae2681d,DISK]]
2024-11-24T04:28:36,384 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 6d7ca8a8825dc62635b3790ec1de87e1 in 439ms, sequenceid=21, compaction requested=false
2024-11-24T04:28:36,384 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6d7ca8a8825dc62635b3790ec1de87e1:

2024-11-24T04:28:36,384 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=24.4 K, sizeToCheck=16.0 K
2024-11-24T04:28:36,384 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-24T04:28:36,385 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/default/TestLogRolling-testSlowSyncLogRolling/6d7ca8a8825dc62635b3790ec1de87e1/info/85c72f8e829f4914ab86a7061708e885 because midkey is the same as first or last row
2024-11-24T04:28:38,150 INFO  [FSHLog-0-hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165-prefix:fadb2fe0ba32,41193,1732422476559 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34465,DS-f52c0a13-f8de-439e-9980-ba843e0b1c2f,DISK], DatanodeInfoWithStorage[127.0.0.1:34329,DS-576af977-24e7-407d-9d91-3882eae2681d,DISK]]
2024-11-24T04:28:38,633 INFO  [master/fadb2fe0ba32:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-11-24T04:28:38,633 INFO  [master/fadb2fe0ba32:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-11-24T04:28:40,354 INFO  [FSHLog-0-hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165-prefix:fadb2fe0ba32,41193,1732422476559 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34465,DS-f52c0a13-f8de-439e-9980-ba843e0b1c2f,DISK], DatanodeInfoWithStorage[127.0.0.1:34329,DS-576af977-24e7-407d-9d91-3882eae2681d,DISK]]
2024-11-24T04:28:40,357 WARN  [FSHLog-0-hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165-prefix:fadb2fe0ba32,41193,1732422476559 {}] wal.AbstractFSWAL(2201): Requesting log roll because we exceeded slow sync threshold; count=8, threshold=5, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34465,DS-f52c0a13-f8de-439e-9980-ba843e0b1c2f,DISK], DatanodeInfoWithStorage[127.0.0.1:34329,DS-576af977-24e7-407d-9d91-3882eae2681d,DISK]]
2024-11-24T04:28:40,358 DEBUG [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog fadb2fe0ba32%2C41193%2C1732422476559:(num 1732422508919) roll requested
2024-11-24T04:28:40,359 INFO  [regionserver/fadb2fe0ba32:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor fadb2fe0ba32%2C41193%2C1732422476559.1732422520359
2024-11-24T04:28:40,570 INFO  [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 208 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34465,DS-f52c0a13-f8de-439e-9980-ba843e0b1c2f,DISK], DatanodeInfoWithStorage[127.0.0.1:34329,DS-576af977-24e7-407d-9d91-3882eae2681d,DISK]]
2024-11-24T04:28:40,570 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:28:40,570 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:28:40,570 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:28:40,571 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:28:40,571 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:28:40,571 INFO  [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/WALs/fadb2fe0ba32,41193,1732422476559/fadb2fe0ba32%2C41193%2C1732422476559.1732422508919 with entries=8, filesize=7.55 KB; new WAL /user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/WALs/fadb2fe0ba32,41193,1732422476559/fadb2fe0ba32%2C41193%2C1732422476559.1732422520359
2024-11-24T04:28:40,574 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741839_1015 (size=7739)
2024-11-24T04:28:40,575 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34465 is added to blk_1073741839_1015 (size=7739)
2024-11-24T04:28:40,580 DEBUG [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38793:38793),(127.0.0.1/127.0.0.1:42911:42911)]
2024-11-24T04:28:40,581 DEBUG [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/WALs/fadb2fe0ba32,41193,1732422476559/fadb2fe0ba32%2C41193%2C1732422476559.1732422508919 is not closed yet, will try archiving it next time
2024-11-24T04:28:40,581 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/WALs/fadb2fe0ba32,41193,1732422476559/fadb2fe0ba32%2C41193%2C1732422476559.1732422488854 to hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/oldWALs/fadb2fe0ba32%2C41193%2C1732422476559.1732422488854
2024-11-24T04:28:42,559 INFO  [FSHLog-0-hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165-prefix:fadb2fe0ba32,41193,1732422476559 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34465,DS-f52c0a13-f8de-439e-9980-ba843e0b1c2f,DISK], DatanodeInfoWithStorage[127.0.0.1:34329,DS-576af977-24e7-407d-9d91-3882eae2681d,DISK]]
2024-11-24T04:28:44,193 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 6d7ca8a8825dc62635b3790ec1de87e1, had cached 0 bytes from a total of 25018
2024-11-24T04:28:44,762 INFO  [FSHLog-0-hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165-prefix:fadb2fe0ba32,41193,1732422476559 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34465,DS-f52c0a13-f8de-439e-9980-ba843e0b1c2f,DISK], DatanodeInfoWithStorage[127.0.0.1:34329,DS-576af977-24e7-407d-9d91-3882eae2681d,DISK]]
2024-11-24T04:28:46,967 INFO  [FSHLog-0-hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165-prefix:fadb2fe0ba32,41193,1732422476559 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34465,DS-f52c0a13-f8de-439e-9980-ba843e0b1c2f,DISK], DatanodeInfoWithStorage[127.0.0.1:34329,DS-576af977-24e7-407d-9d91-3882eae2681d,DISK]]
2024-11-24T04:28:49,172 INFO  [FSHLog-0-hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165-prefix:fadb2fe0ba32,41193,1732422476559 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34465,DS-f52c0a13-f8de-439e-9980-ba843e0b1c2f,DISK], DatanodeInfoWithStorage[127.0.0.1:34329,DS-576af977-24e7-407d-9d91-3882eae2681d,DISK]]
2024-11-24T04:28:51,174 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-11-24T04:28:51,175 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor fadb2fe0ba32%2C41193%2C1732422476559.1732422531175
2024-11-24T04:28:54,967 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-24T04:28:56,184 INFO  [Time-limited test {}] wal.AbstractFSWAL(1368): Slow sync cost: 5006 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34465,DS-f52c0a13-f8de-439e-9980-ba843e0b1c2f,DISK], DatanodeInfoWithStorage[127.0.0.1:34329,DS-576af977-24e7-407d-9d91-3882eae2681d,DISK]]
2024-11-24T04:28:56,186 WARN  [Time-limited test {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5006 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34465,DS-f52c0a13-f8de-439e-9980-ba843e0b1c2f,DISK], DatanodeInfoWithStorage[127.0.0.1:34329,DS-576af977-24e7-407d-9d91-3882eae2681d,DISK]]
2024-11-24T04:28:56,186 DEBUG [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog fadb2fe0ba32%2C41193%2C1732422476559:(num 1732422531175) roll requested
2024-11-24T04:28:56,186 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:28:56,186 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:28:56,186 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:28:56,186 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:28:56,186 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:28:56,187 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/WALs/fadb2fe0ba32,41193,1732422476559/fadb2fe0ba32%2C41193%2C1732422476559.1732422520359 with entries=4, filesize=4.63 KB; new WAL /user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/WALs/fadb2fe0ba32,41193,1732422476559/fadb2fe0ba32%2C41193%2C1732422476559.1732422531175
2024-11-24T04:28:56,188 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38793:38793),(127.0.0.1/127.0.0.1:42911:42911)]
2024-11-24T04:28:56,188 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/WALs/fadb2fe0ba32,41193,1732422476559/fadb2fe0ba32%2C41193%2C1732422476559.1732422520359 is not closed yet, will try archiving it next time
2024-11-24T04:28:56,188 INFO  [regionserver/fadb2fe0ba32:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor fadb2fe0ba32%2C41193%2C1732422476559.1732422536188
2024-11-24T04:28:56,189 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741841_1017 (size=4753)
2024-11-24T04:28:56,190 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34465 is added to blk_1073741841_1017 (size=4753)
2024-11-24T04:29:01,192 INFO  [FSHLog-0-hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165-prefix:fadb2fe0ba32,41193,1732422476559 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5001 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34465,DS-f52c0a13-f8de-439e-9980-ba843e0b1c2f,DISK], DatanodeInfoWithStorage[127.0.0.1:34329,DS-576af977-24e7-407d-9d91-3882eae2681d,DISK]]
2024-11-24T04:29:01,192 WARN  [FSHLog-0-hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165-prefix:fadb2fe0ba32,41193,1732422476559 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5001 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34465,DS-f52c0a13-f8de-439e-9980-ba843e0b1c2f,DISK], DatanodeInfoWithStorage[127.0.0.1:34329,DS-576af977-24e7-407d-9d91-3882eae2681d,DISK]]
2024-11-24T04:29:01,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41193 {}] regionserver.HRegion(8855): Flush requested on 6d7ca8a8825dc62635b3790ec1de87e1
2024-11-24T04:29:01,192 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6d7ca8a8825dc62635b3790ec1de87e1 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-24T04:29:01,198 INFO  [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 5006 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34465,DS-f52c0a13-f8de-439e-9980-ba843e0b1c2f,DISK], DatanodeInfoWithStorage[127.0.0.1:34329,DS-576af977-24e7-407d-9d91-3882eae2681d,DISK]]
2024-11-24T04:29:01,198 WARN  [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5006 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34465,DS-f52c0a13-f8de-439e-9980-ba843e0b1c2f,DISK], DatanodeInfoWithStorage[127.0.0.1:34329,DS-576af977-24e7-407d-9d91-3882eae2681d,DISK]]
2024-11-24T04:29:03,193 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-11-24T04:29:06,195 INFO  [FSHLog-0-hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165-prefix:fadb2fe0ba32,41193,1732422476559 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5001 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34465,DS-f52c0a13-f8de-439e-9980-ba843e0b1c2f,DISK], DatanodeInfoWithStorage[127.0.0.1:34329,DS-576af977-24e7-407d-9d91-3882eae2681d,DISK]]
2024-11-24T04:29:06,195 WARN  [FSHLog-0-hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165-prefix:fadb2fe0ba32,41193,1732422476559 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5001 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34465,DS-f52c0a13-f8de-439e-9980-ba843e0b1c2f,DISK], DatanodeInfoWithStorage[127.0.0.1:34329,DS-576af977-24e7-407d-9d91-3882eae2681d,DISK]]
2024-11-24T04:29:06,195 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:06,195 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:06,195 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:06,195 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:06,196 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:06,196 INFO  [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/WALs/fadb2fe0ba32,41193,1732422476559/fadb2fe0ba32%2C41193%2C1732422476559.1732422531175 with entries=2, filesize=1.52 KB; new WAL /user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/WALs/fadb2fe0ba32,41193,1732422476559/fadb2fe0ba32%2C41193%2C1732422476559.1732422536188
2024-11-24T04:29:06,197 DEBUG [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38793:38793),(127.0.0.1/127.0.0.1:42911:42911)]
2024-11-24T04:29:06,197 DEBUG [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/WALs/fadb2fe0ba32,41193,1732422476559/fadb2fe0ba32%2C41193%2C1732422476559.1732422531175 is not closed yet, will try archiving it next time
2024-11-24T04:29:06,197 DEBUG [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog fadb2fe0ba32%2C41193%2C1732422476559:(num 1732422536188) roll requested
2024-11-24T04:29:06,198 INFO  [regionserver/fadb2fe0ba32:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor fadb2fe0ba32%2C41193%2C1732422476559.1732422546197
2024-11-24T04:29:06,198 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741842_1018 (size=1569)
2024-11-24T04:29:06,199 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34465 is added to blk_1073741842_1018 (size=1569)
2024-11-24T04:29:06,202 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/default/TestLogRolling-testSlowSyncLogRolling/6d7ca8a8825dc62635b3790ec1de87e1/.tmp/info/e4e7c8adf06449beb34429fc54f6197b is 1080, key is row0015/info:/1732422517948/Put/seqid=0
2024-11-24T04:29:06,210 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741844_1020 (size=12509)
2024-11-24T04:29:06,210 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34465 is added to blk_1073741844_1020 (size=12509)
2024-11-24T04:29:06,211 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/default/TestLogRolling-testSlowSyncLogRolling/6d7ca8a8825dc62635b3790ec1de87e1/.tmp/info/e4e7c8adf06449beb34429fc54f6197b
2024-11-24T04:29:06,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/default/TestLogRolling-testSlowSyncLogRolling/6d7ca8a8825dc62635b3790ec1de87e1/.tmp/info/e4e7c8adf06449beb34429fc54f6197b as hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/default/TestLogRolling-testSlowSyncLogRolling/6d7ca8a8825dc62635b3790ec1de87e1/info/e4e7c8adf06449beb34429fc54f6197b
2024-11-24T04:29:06,231 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/default/TestLogRolling-testSlowSyncLogRolling/6d7ca8a8825dc62635b3790ec1de87e1/info/e4e7c8adf06449beb34429fc54f6197b, entries=7, sequenceid=31, filesize=12.2 K
2024-11-24T04:29:11,206 INFO  [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 5005 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34465,DS-f52c0a13-f8de-439e-9980-ba843e0b1c2f,DISK], DatanodeInfoWithStorage[127.0.0.1:34329,DS-576af977-24e7-407d-9d91-3882eae2681d,DISK]]
2024-11-24T04:29:11,206 WARN  [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5005 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34465,DS-f52c0a13-f8de-439e-9980-ba843e0b1c2f,DISK], DatanodeInfoWithStorage[127.0.0.1:34329,DS-576af977-24e7-407d-9d91-3882eae2681d,DISK]]
2024-11-24T04:29:11,233 INFO  [FSHLog-0-hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165-prefix:fadb2fe0ba32,41193,1732422476559 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5001 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34465,DS-f52c0a13-f8de-439e-9980-ba843e0b1c2f,DISK], DatanodeInfoWithStorage[127.0.0.1:34329,DS-576af977-24e7-407d-9d91-3882eae2681d,DISK]]
2024-11-24T04:29:11,233 WARN  [FSHLog-0-hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165-prefix:fadb2fe0ba32,41193,1732422476559 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5001 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34465,DS-f52c0a13-f8de-439e-9980-ba843e0b1c2f,DISK], DatanodeInfoWithStorage[127.0.0.1:34329,DS-576af977-24e7-407d-9d91-3882eae2681d,DISK]]
2024-11-24T04:29:11,233 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 6d7ca8a8825dc62635b3790ec1de87e1 in 10041ms, sequenceid=31, compaction requested=true
2024-11-24T04:29:11,233 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:11,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6d7ca8a8825dc62635b3790ec1de87e1:

2024-11-24T04:29:11,233 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:11,233 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=36.6 K, sizeToCheck=16.0 K
2024-11-24T04:29:11,233 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:11,233 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-24T04:29:11,234 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:11,234 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/default/TestLogRolling-testSlowSyncLogRolling/6d7ca8a8825dc62635b3790ec1de87e1/info/85c72f8e829f4914ab86a7061708e885 because midkey is the same as first or last row
2024-11-24T04:29:11,234 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:11,234 INFO  [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/WALs/fadb2fe0ba32,41193,1732422476559/fadb2fe0ba32%2C41193%2C1732422476559.1732422536188 with entries=1, filesize=430 B; new WAL /user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/WALs/fadb2fe0ba32,41193,1732422476559/fadb2fe0ba32%2C41193%2C1732422476559.1732422546197
2024-11-24T04:29:11,235 DEBUG [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38793:38793),(127.0.0.1/127.0.0.1:42911:42911)]
2024-11-24T04:29:11,235 DEBUG [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/WALs/fadb2fe0ba32,41193,1732422476559/fadb2fe0ba32%2C41193%2C1732422476559.1732422536188 is not closed yet, will try archiving it next time
2024-11-24T04:29:11,235 DEBUG [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog fadb2fe0ba32%2C41193%2C1732422476559:(num 1732422551235) roll requested
2024-11-24T04:29:11,235 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/WALs/fadb2fe0ba32,41193,1732422476559/fadb2fe0ba32%2C41193%2C1732422476559.1732422508919 to hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/oldWALs/fadb2fe0ba32%2C41193%2C1732422476559.1732422508919
2024-11-24T04:29:11,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d7ca8a8825dc62635b3790ec1de87e1:info, priority=-2147483648, current under compaction store size is 1
2024-11-24T04:29:11,235 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor fadb2fe0ba32%2C41193%2C1732422476559.1732422551235
2024-11-24T04:29:11,237 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741843_1019 (size=438)
2024-11-24T04:29:11,237 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34465 is added to blk_1073741843_1019 (size=438)
2024-11-24T04:29:11,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-24T04:29:11,238 DEBUG [RS:0;fadb2fe0ba32:41193-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-24T04:29:11,239 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/WALs/fadb2fe0ba32,41193,1732422476559/fadb2fe0ba32%2C41193%2C1732422476559.1732422520359 to hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/oldWALs/fadb2fe0ba32%2C41193%2C1732422476559.1732422520359
2024-11-24T04:29:11,241 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/WALs/fadb2fe0ba32,41193,1732422476559/fadb2fe0ba32%2C41193%2C1732422476559.1732422531175 to hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/oldWALs/fadb2fe0ba32%2C41193%2C1732422476559.1732422531175
2024-11-24T04:29:11,241 DEBUG [RS:0;fadb2fe0ba32:41193-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 37527 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-24T04:29:11,243 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/WALs/fadb2fe0ba32,41193,1732422476559/fadb2fe0ba32%2C41193%2C1732422476559.1732422536188 to hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/oldWALs/fadb2fe0ba32%2C41193%2C1732422476559.1732422536188
2024-11-24T04:29:11,243 DEBUG [RS:0;fadb2fe0ba32:41193-shortCompactions-0 {}] regionserver.HStore(1541): 6d7ca8a8825dc62635b3790ec1de87e1/info is initiating minor compaction (all files)
2024-11-24T04:29:11,243 INFO  [RS:0;fadb2fe0ba32:41193-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d7ca8a8825dc62635b3790ec1de87e1/info in TestLogRolling-testSlowSyncLogRolling,,1732422478756.6d7ca8a8825dc62635b3790ec1de87e1.
2024-11-24T04:29:11,244 INFO  [RS:0;fadb2fe0ba32:41193-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/default/TestLogRolling-testSlowSyncLogRolling/6d7ca8a8825dc62635b3790ec1de87e1/info/85c72f8e829f4914ab86a7061708e885, hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/default/TestLogRolling-testSlowSyncLogRolling/6d7ca8a8825dc62635b3790ec1de87e1/info/f8e125503e1c4e0ab9eb20ed7cde1e09, hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/default/TestLogRolling-testSlowSyncLogRolling/6d7ca8a8825dc62635b3790ec1de87e1/info/e4e7c8adf06449beb34429fc54f6197b] into tmpdir=hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/default/TestLogRolling-testSlowSyncLogRolling/6d7ca8a8825dc62635b3790ec1de87e1/.tmp, totalSize=36.6 K
2024-11-24T04:29:11,245 DEBUG [RS:0;fadb2fe0ba32:41193-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85c72f8e829f4914ab86a7061708e885, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732422488878
2024-11-24T04:29:11,245 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:11,245 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:11,246 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:11,246 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:11,246 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:11,246 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/WALs/fadb2fe0ba32,41193,1732422476559/fadb2fe0ba32%2C41193%2C1732422476559.1732422546197 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/WALs/fadb2fe0ba32,41193,1732422476559/fadb2fe0ba32%2C41193%2C1732422476559.1732422551235
2024-11-24T04:29:11,246 DEBUG [RS:0;fadb2fe0ba32:41193-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8e125503e1c4e0ab9eb20ed7cde1e09, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=21, earliestPutTs=1732422502909
2024-11-24T04:29:11,247 DEBUG [RS:0;fadb2fe0ba32:41193-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4e7c8adf06449beb34429fc54f6197b, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=31, earliestPutTs=1732422517948
2024-11-24T04:29:11,247 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38793:38793),(127.0.0.1/127.0.0.1:42911:42911)]
2024-11-24T04:29:11,247 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/WALs/fadb2fe0ba32,41193,1732422476559/fadb2fe0ba32%2C41193%2C1732422476559.1732422546197 is not closed yet, will try archiving it next time
2024-11-24T04:29:11,248 INFO  [regionserver/fadb2fe0ba32:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor fadb2fe0ba32%2C41193%2C1732422476559.1732422551247
2024-11-24T04:29:11,249 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741845_1021 (size=93)
2024-11-24T04:29:11,250 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34465 is added to blk_1073741845_1021 (size=93)
2024-11-24T04:29:11,251 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/WALs/fadb2fe0ba32,41193,1732422476559/fadb2fe0ba32%2C41193%2C1732422476559.1732422546197 to hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/oldWALs/fadb2fe0ba32%2C41193%2C1732422476559.1732422546197
2024-11-24T04:29:11,260 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:11,260 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:11,261 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:11,261 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:11,261 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:11,261 INFO  [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/WALs/fadb2fe0ba32,41193,1732422476559/fadb2fe0ba32%2C41193%2C1732422476559.1732422551235 with entries=1, filesize=1.22 KB; new WAL /user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/WALs/fadb2fe0ba32,41193,1732422476559/fadb2fe0ba32%2C41193%2C1732422476559.1732422551247
2024-11-24T04:29:11,263 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741846_1022 (size=1258)
2024-11-24T04:29:11,264 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34465 is added to blk_1073741846_1022 (size=1258)
2024-11-24T04:29:11,269 DEBUG [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38793:38793),(127.0.0.1/127.0.0.1:42911:42911)]
2024-11-24T04:29:11,288 INFO  [RS:0;fadb2fe0ba32:41193-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d7ca8a8825dc62635b3790ec1de87e1#info#compaction#3 average throughput is 10.77 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-24T04:29:11,290 DEBUG [RS:0;fadb2fe0ba32:41193-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/default/TestLogRolling-testSlowSyncLogRolling/6d7ca8a8825dc62635b3790ec1de87e1/.tmp/info/44d16f965d244dcda8ce4adf25a693c1 is 1080, key is row0001/info:/1732422488878/Put/seqid=0
2024-11-24T04:29:11,296 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741848_1024 (size=27710)
2024-11-24T04:29:11,296 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34465 is added to blk_1073741848_1024 (size=27710)
2024-11-24T04:29:11,307 DEBUG [RS:0;fadb2fe0ba32:41193-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/default/TestLogRolling-testSlowSyncLogRolling/6d7ca8a8825dc62635b3790ec1de87e1/.tmp/info/44d16f965d244dcda8ce4adf25a693c1 as hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/default/TestLogRolling-testSlowSyncLogRolling/6d7ca8a8825dc62635b3790ec1de87e1/info/44d16f965d244dcda8ce4adf25a693c1
2024-11-24T04:29:11,326 INFO  [RS:0;fadb2fe0ba32:41193-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d7ca8a8825dc62635b3790ec1de87e1/info of 6d7ca8a8825dc62635b3790ec1de87e1 into 44d16f965d244dcda8ce4adf25a693c1(size=27.1 K), total size for store is 27.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-24T04:29:11,326 DEBUG [RS:0;fadb2fe0ba32:41193-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d7ca8a8825dc62635b3790ec1de87e1:

2024-11-24T04:29:11,328 INFO  [RS:0;fadb2fe0ba32:41193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testSlowSyncLogRolling,,1732422478756.6d7ca8a8825dc62635b3790ec1de87e1., storeName=6d7ca8a8825dc62635b3790ec1de87e1/info, priority=13, startTime=1732422551235; duration=0sec
2024-11-24T04:29:11,329 DEBUG [RS:0;fadb2fe0ba32:41193-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K
2024-11-24T04:29:11,329 DEBUG [RS:0;fadb2fe0ba32:41193-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-24T04:29:11,329 DEBUG [RS:0;fadb2fe0ba32:41193-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/default/TestLogRolling-testSlowSyncLogRolling/6d7ca8a8825dc62635b3790ec1de87e1/info/44d16f965d244dcda8ce4adf25a693c1 because midkey is the same as first or last row
2024-11-24T04:29:11,329 DEBUG [RS:0;fadb2fe0ba32:41193-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K
2024-11-24T04:29:11,329 DEBUG [RS:0;fadb2fe0ba32:41193-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-24T04:29:11,329 DEBUG [RS:0;fadb2fe0ba32:41193-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/default/TestLogRolling-testSlowSyncLogRolling/6d7ca8a8825dc62635b3790ec1de87e1/info/44d16f965d244dcda8ce4adf25a693c1 because midkey is the same as first or last row
2024-11-24T04:29:11,329 DEBUG [RS:0;fadb2fe0ba32:41193-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K
2024-11-24T04:29:11,329 DEBUG [RS:0;fadb2fe0ba32:41193-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-24T04:29:11,329 DEBUG [RS:0;fadb2fe0ba32:41193-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/default/TestLogRolling-testSlowSyncLogRolling/6d7ca8a8825dc62635b3790ec1de87e1/info/44d16f965d244dcda8ce4adf25a693c1 because midkey is the same as first or last row
2024-11-24T04:29:11,330 DEBUG [RS:0;fadb2fe0ba32:41193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-24T04:29:11,330 DEBUG [RS:0;fadb2fe0ba32:41193-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d7ca8a8825dc62635b3790ec1de87e1:info
2024-11-24T04:29:23,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41193 {}] regionserver.HRegion(8855): Flush requested on 6d7ca8a8825dc62635b3790ec1de87e1
2024-11-24T04:29:23,275 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6d7ca8a8825dc62635b3790ec1de87e1 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-24T04:29:23,281 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/default/TestLogRolling-testSlowSyncLogRolling/6d7ca8a8825dc62635b3790ec1de87e1/.tmp/info/fa8e093dc40e4cca8515eac4ba1198fe is 1080, key is row0022/info:/1732422551249/Put/seqid=0
2024-11-24T04:29:23,288 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34465 is added to blk_1073741849_1025 (size=12509)
2024-11-24T04:29:23,288 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741849_1025 (size=12509)
2024-11-24T04:29:23,289 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/default/TestLogRolling-testSlowSyncLogRolling/6d7ca8a8825dc62635b3790ec1de87e1/.tmp/info/fa8e093dc40e4cca8515eac4ba1198fe
2024-11-24T04:29:23,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/default/TestLogRolling-testSlowSyncLogRolling/6d7ca8a8825dc62635b3790ec1de87e1/.tmp/info/fa8e093dc40e4cca8515eac4ba1198fe as hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/default/TestLogRolling-testSlowSyncLogRolling/6d7ca8a8825dc62635b3790ec1de87e1/info/fa8e093dc40e4cca8515eac4ba1198fe
2024-11-24T04:29:23,307 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/default/TestLogRolling-testSlowSyncLogRolling/6d7ca8a8825dc62635b3790ec1de87e1/info/fa8e093dc40e4cca8515eac4ba1198fe, entries=7, sequenceid=42, filesize=12.2 K
2024-11-24T04:29:23,308 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 6d7ca8a8825dc62635b3790ec1de87e1 in 34ms, sequenceid=42, compaction requested=false
2024-11-24T04:29:23,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6d7ca8a8825dc62635b3790ec1de87e1:

2024-11-24T04:29:23,309 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=39.3 K, sizeToCheck=16.0 K
2024-11-24T04:29:23,309 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-24T04:29:23,309 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/default/TestLogRolling-testSlowSyncLogRolling/6d7ca8a8825dc62635b3790ec1de87e1/info/44d16f965d244dcda8ce4adf25a693c1 because midkey is the same as first or last row
2024-11-24T04:29:24,968 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-24T04:29:29,194 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 6d7ca8a8825dc62635b3790ec1de87e1, had cached 0 bytes from a total of 40219
2024-11-24T04:29:31,287 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-24T04:29:31,287 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-24T04:29:31,287 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-24T04:29:31,292 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T04:29:31,293 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T04:29:31,293 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-24T04:29:31,293 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-24T04:29:31,293 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1731086434, stopped=false
2024-11-24T04:29:31,294 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=fadb2fe0ba32,43529,1732422475859
2024-11-24T04:29:31,296 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41193-0x101483b07a20001, quorum=127.0.0.1:57793, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-24T04:29:31,296 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43529-0x101483b07a20000, quorum=127.0.0.1:57793, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-24T04:29:31,296 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41193-0x101483b07a20001, quorum=127.0.0.1:57793, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:29:31,296 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43529-0x101483b07a20000, quorum=127.0.0.1:57793, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:29:31,296 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-24T04:29:31,296 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-24T04:29:31,296 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-24T04:29:31,296 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T04:29:31,296 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43529-0x101483b07a20000, quorum=127.0.0.1:57793, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-24T04:29:31,297 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'fadb2fe0ba32,41193,1732422476559' *****
2024-11-24T04:29:31,297 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-24T04:29:31,297 INFO  [RS:0;fadb2fe0ba32:41193 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-24T04:29:31,297 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41193-0x101483b07a20001, quorum=127.0.0.1:57793, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-24T04:29:31,297 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-24T04:29:31,297 INFO  [RS:0;fadb2fe0ba32:41193 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-24T04:29:31,298 INFO  [RS:0;fadb2fe0ba32:41193 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-24T04:29:31,298 INFO  [RS:0;fadb2fe0ba32:41193 {}] regionserver.HRegionServer(3091): Received CLOSE for 6d7ca8a8825dc62635b3790ec1de87e1
2024-11-24T04:29:31,299 INFO  [RS:0;fadb2fe0ba32:41193 {}] regionserver.HRegionServer(959): stopping server fadb2fe0ba32,41193,1732422476559
2024-11-24T04:29:31,299 INFO  [RS:0;fadb2fe0ba32:41193 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-24T04:29:31,299 INFO  [RS:0;fadb2fe0ba32:41193 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;fadb2fe0ba32:41193.
2024-11-24T04:29:31,299 DEBUG [RS:0;fadb2fe0ba32:41193 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-24T04:29:31,299 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6d7ca8a8825dc62635b3790ec1de87e1, disabling compactions & flushes
2024-11-24T04:29:31,299 DEBUG [RS:0;fadb2fe0ba32:41193 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T04:29:31,299 INFO  [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testSlowSyncLogRolling,,1732422478756.6d7ca8a8825dc62635b3790ec1de87e1.
2024-11-24T04:29:31,299 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testSlowSyncLogRolling,,1732422478756.6d7ca8a8825dc62635b3790ec1de87e1.
2024-11-24T04:29:31,299 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1732422478756.6d7ca8a8825dc62635b3790ec1de87e1. after waiting 0 ms
2024-11-24T04:29:31,299 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1732422478756.6d7ca8a8825dc62635b3790ec1de87e1.
2024-11-24T04:29:31,299 INFO  [RS:0;fadb2fe0ba32:41193 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-24T04:29:31,299 INFO  [RS:0;fadb2fe0ba32:41193 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-24T04:29:31,299 INFO  [RS:0;fadb2fe0ba32:41193 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-24T04:29:31,299 INFO  [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 6d7ca8a8825dc62635b3790ec1de87e1 1/1 column families, dataSize=3.15 KB heapSize=3.63 KB
2024-11-24T04:29:31,299 INFO  [RS:0;fadb2fe0ba32:41193 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-24T04:29:31,300 INFO  [RS:0;fadb2fe0ba32:41193 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-11-24T04:29:31,300 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-24T04:29:31,300 DEBUG [RS:0;fadb2fe0ba32:41193 {}] regionserver.HRegionServer(1325): Online Regions={6d7ca8a8825dc62635b3790ec1de87e1=TestLogRolling-testSlowSyncLogRolling,,1732422478756.6d7ca8a8825dc62635b3790ec1de87e1., 1588230740=hbase:meta,,1.1588230740}
2024-11-24T04:29:31,300 INFO  [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-24T04:29:31,300 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-24T04:29:31,300 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-24T04:29:31,300 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-24T04:29:31,301 DEBUG [RS:0;fadb2fe0ba32:41193 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 6d7ca8a8825dc62635b3790ec1de87e1
2024-11-24T04:29:31,301 INFO  [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.65 KB heapSize=3.67 KB
2024-11-24T04:29:31,310 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/default/TestLogRolling-testSlowSyncLogRolling/6d7ca8a8825dc62635b3790ec1de87e1/.tmp/info/47b770cd862841dca01c026be52dcfab is 1080, key is row0029/info:/1732422565277/Put/seqid=0
2024-11-24T04:29:31,338 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/hbase/meta/1588230740/.tmp/info/5b852b16924747dcb44f00a908ca2b27 is 195, key is TestLogRolling-testSlowSyncLogRolling,,1732422478756.6d7ca8a8825dc62635b3790ec1de87e1./info:regioninfo/1732422479219/Put/seqid=0
2024-11-24T04:29:31,351 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741850_1026 (size=8193)
2024-11-24T04:29:31,351 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34465 is added to blk_1073741850_1026 (size=8193)
2024-11-24T04:29:31,358 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741851_1027 (size=7016)
2024-11-24T04:29:31,358 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34465 is added to blk_1073741851_1027 (size=7016)
2024-11-24T04:29:31,359 INFO  [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.45 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/hbase/meta/1588230740/.tmp/info/5b852b16924747dcb44f00a908ca2b27
2024-11-24T04:29:31,406 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/hbase/meta/1588230740/.tmp/ns/4a31442552274709abbbf44265d6d696 is 43, key is default/ns:d/1732422478533/Put/seqid=0
2024-11-24T04:29:31,424 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34465 is added to blk_1073741852_1028 (size=5153)
2024-11-24T04:29:31,424 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741852_1028 (size=5153)
2024-11-24T04:29:31,425 INFO  [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/hbase/meta/1588230740/.tmp/ns/4a31442552274709abbbf44265d6d696
2024-11-24T04:29:31,461 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/hbase/meta/1588230740/.tmp/table/c5ec5b823b5d46b495707dda60334ca4 is 73, key is TestLogRolling-testSlowSyncLogRolling/table:state/1732422479237/Put/seqid=0
2024-11-24T04:29:31,475 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34465 is added to blk_1073741853_1029 (size=5396)
2024-11-24T04:29:31,476 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741853_1029 (size=5396)
2024-11-24T04:29:31,476 INFO  [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=138 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/hbase/meta/1588230740/.tmp/table/c5ec5b823b5d46b495707dda60334ca4
2024-11-24T04:29:31,485 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/hbase/meta/1588230740/.tmp/info/5b852b16924747dcb44f00a908ca2b27 as hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/hbase/meta/1588230740/info/5b852b16924747dcb44f00a908ca2b27
2024-11-24T04:29:31,493 INFO  [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/hbase/meta/1588230740/info/5b852b16924747dcb44f00a908ca2b27, entries=10, sequenceid=11, filesize=6.9 K
2024-11-24T04:29:31,495 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/hbase/meta/1588230740/.tmp/ns/4a31442552274709abbbf44265d6d696 as hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/hbase/meta/1588230740/ns/4a31442552274709abbbf44265d6d696
2024-11-24T04:29:31,501 DEBUG [RS:0;fadb2fe0ba32:41193 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 6d7ca8a8825dc62635b3790ec1de87e1
2024-11-24T04:29:31,502 INFO  [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/hbase/meta/1588230740/ns/4a31442552274709abbbf44265d6d696, entries=2, sequenceid=11, filesize=5.0 K
2024-11-24T04:29:31,503 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/hbase/meta/1588230740/.tmp/table/c5ec5b823b5d46b495707dda60334ca4 as hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/hbase/meta/1588230740/table/c5ec5b823b5d46b495707dda60334ca4
2024-11-24T04:29:31,510 INFO  [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/hbase/meta/1588230740/table/c5ec5b823b5d46b495707dda60334ca4, entries=2, sequenceid=11, filesize=5.3 K
2024-11-24T04:29:31,512 INFO  [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.65 KB/1692, heapSize ~3.38 KB/3456, currentSize=0 B/0 for 1588230740 in 212ms, sequenceid=11, compaction requested=false
2024-11-24T04:29:31,519 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-11-24T04:29:31,521 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-24T04:29:31,521 INFO  [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-24T04:29:31,521 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732422571300Running coprocessor pre-close hooks at 1732422571300Disabling compacts and flushes for region at 1732422571300Disabling writes for close at 1732422571300Obtaining lock to block concurrent updates at 1732422571301 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1732422571301Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1692, getHeapSize=3696, getOffHeapSize=0, getCellsCount=14 at 1732422571301Flushing stores of hbase:meta,,1.1588230740 at 1732422571303 (+2 ms)Flushing 1588230740/info: creating writer at 1732422571303Flushing 1588230740/info: appending metadata at 1732422571338 (+35 ms)Flushing 1588230740/info: closing flushed file at 1732422571338Flushing 1588230740/ns: creating writer at 1732422571382 (+44 ms)Flushing 1588230740/ns: appending metadata at 1732422571406 (+24 ms)Flushing 1588230740/ns: closing flushed file at 1732422571406Flushing 1588230740/table: creating writer at 1732422571436 (+30 ms)Flushing 1588230740/table: appending metadata at 1732422571460 (+24 ms)Flushing 1588230740/table: closing flushed file at 1732422571460Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@39841749: reopening flushed file at 1732422571484 (+24 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@64c7c434: reopening flushed file at 1732422571494 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@14078855: reopening flushed file at 1732422571502 (+8 ms)Finished flush of dataSize ~1.65 KB/1692, heapSize ~3.38 KB/3456, currentSize=0 B/0 for 1588230740 in 212ms, sequenceid=11, compaction requested=false at 1732422571512 (+10 ms)Writing region close event to WAL at 1732422571514 (+2 ms)Running coprocessor post-close hooks at 1732422571519 (+5 ms)Closed at 1732422571521 (+2 ms)
2024-11-24T04:29:31,522 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-24T04:29:31,669 INFO  [regionserver/fadb2fe0ba32:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-24T04:29:31,701 DEBUG [RS:0;fadb2fe0ba32:41193 {}] regionserver.HRegionServer(1351): Waiting on 6d7ca8a8825dc62635b3790ec1de87e1
2024-11-24T04:29:31,724 INFO  [regionserver/fadb2fe0ba32:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-11-24T04:29:31,724 INFO  [regionserver/fadb2fe0ba32:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-11-24T04:29:31,752 INFO  [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.15 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/default/TestLogRolling-testSlowSyncLogRolling/6d7ca8a8825dc62635b3790ec1de87e1/.tmp/info/47b770cd862841dca01c026be52dcfab
2024-11-24T04:29:31,761 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/default/TestLogRolling-testSlowSyncLogRolling/6d7ca8a8825dc62635b3790ec1de87e1/.tmp/info/47b770cd862841dca01c026be52dcfab as hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/default/TestLogRolling-testSlowSyncLogRolling/6d7ca8a8825dc62635b3790ec1de87e1/info/47b770cd862841dca01c026be52dcfab
2024-11-24T04:29:31,768 INFO  [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/default/TestLogRolling-testSlowSyncLogRolling/6d7ca8a8825dc62635b3790ec1de87e1/info/47b770cd862841dca01c026be52dcfab, entries=3, sequenceid=48, filesize=8.0 K
2024-11-24T04:29:31,770 INFO  [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 6d7ca8a8825dc62635b3790ec1de87e1 in 470ms, sequenceid=48, compaction requested=true
2024-11-24T04:29:31,770 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1732422478756.6d7ca8a8825dc62635b3790ec1de87e1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/default/TestLogRolling-testSlowSyncLogRolling/6d7ca8a8825dc62635b3790ec1de87e1/info/85c72f8e829f4914ab86a7061708e885, hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/default/TestLogRolling-testSlowSyncLogRolling/6d7ca8a8825dc62635b3790ec1de87e1/info/f8e125503e1c4e0ab9eb20ed7cde1e09, hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/default/TestLogRolling-testSlowSyncLogRolling/6d7ca8a8825dc62635b3790ec1de87e1/info/e4e7c8adf06449beb34429fc54f6197b] to archive
2024-11-24T04:29:31,773 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1732422478756.6d7ca8a8825dc62635b3790ec1de87e1.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-24T04:29:31,776 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1732422478756.6d7ca8a8825dc62635b3790ec1de87e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/default/TestLogRolling-testSlowSyncLogRolling/6d7ca8a8825dc62635b3790ec1de87e1/info/85c72f8e829f4914ab86a7061708e885 to hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/archive/data/default/TestLogRolling-testSlowSyncLogRolling/6d7ca8a8825dc62635b3790ec1de87e1/info/85c72f8e829f4914ab86a7061708e885
2024-11-24T04:29:31,778 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1732422478756.6d7ca8a8825dc62635b3790ec1de87e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/default/TestLogRolling-testSlowSyncLogRolling/6d7ca8a8825dc62635b3790ec1de87e1/info/f8e125503e1c4e0ab9eb20ed7cde1e09 to hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/archive/data/default/TestLogRolling-testSlowSyncLogRolling/6d7ca8a8825dc62635b3790ec1de87e1/info/f8e125503e1c4e0ab9eb20ed7cde1e09
2024-11-24T04:29:31,779 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1732422478756.6d7ca8a8825dc62635b3790ec1de87e1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/default/TestLogRolling-testSlowSyncLogRolling/6d7ca8a8825dc62635b3790ec1de87e1/info/e4e7c8adf06449beb34429fc54f6197b to hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/archive/data/default/TestLogRolling-testSlowSyncLogRolling/6d7ca8a8825dc62635b3790ec1de87e1/info/e4e7c8adf06449beb34429fc54f6197b
2024-11-24T04:29:31,791 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1732422478756.6d7ca8a8825dc62635b3790ec1de87e1.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=fadb2fe0ba32:43529 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 16 more
2024-11-24T04:29:31,795 WARN  [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1732422478756.6d7ca8a8825dc62635b3790ec1de87e1.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [85c72f8e829f4914ab86a7061708e885=12509, f8e125503e1c4e0ab9eb20ed7cde1e09=12509, e4e7c8adf06449beb34429fc54f6197b=12509]
2024-11-24T04:29:31,801 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/data/default/TestLogRolling-testSlowSyncLogRolling/6d7ca8a8825dc62635b3790ec1de87e1/recovered.edits/51.seqid, newMaxSeqId=51, maxSeqId=1
2024-11-24T04:29:31,802 INFO  [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testSlowSyncLogRolling,,1732422478756.6d7ca8a8825dc62635b3790ec1de87e1.
2024-11-24T04:29:31,802 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6d7ca8a8825dc62635b3790ec1de87e1:
Waiting for close lock at 1732422571299Running coprocessor pre-close hooks at 1732422571299Disabling compacts and flushes for region at 1732422571299Disabling writes for close at 1732422571299Obtaining lock to block concurrent updates at 1732422571299Preparing flush snapshotting stores in 6d7ca8a8825dc62635b3790ec1de87e1 at 1732422571299Finished memstore snapshotting TestLogRolling-testSlowSyncLogRolling,,1732422478756.6d7ca8a8825dc62635b3790ec1de87e1., syncing WAL and waiting on mvcc, flushsize=dataSize=3228, getHeapSize=3696, getOffHeapSize=0, getCellsCount=3 at 1732422571300 (+1 ms)Flushing stores of TestLogRolling-testSlowSyncLogRolling,,1732422478756.6d7ca8a8825dc62635b3790ec1de87e1. at 1732422571301 (+1 ms)Flushing 6d7ca8a8825dc62635b3790ec1de87e1/info: creating writer at 1732422571302 (+1 ms)Flushing 6d7ca8a8825dc62635b3790ec1de87e1/info: appending metadata at 1732422571309 (+7 ms)Flushing 6d7ca8a8825dc62635b3790ec1de87e1/info: closing flushed file at 1732422571309Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@f5ee263: reopening flushed file at 1732422571760 (+451 ms)Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 6d7ca8a8825dc62635b3790ec1de87e1 in 470ms, sequenceid=48, compaction requested=true at 1732422571770 (+10 ms)Writing region close event to WAL at 1732422571796 (+26 ms)Running coprocessor post-close hooks at 1732422571802 (+6 ms)Closed at 1732422571802
2024-11-24T04:29:31,802 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testSlowSyncLogRolling,,1732422478756.6d7ca8a8825dc62635b3790ec1de87e1.
2024-11-24T04:29:31,901 INFO  [RS:0;fadb2fe0ba32:41193 {}] regionserver.HRegionServer(976): stopping server fadb2fe0ba32,41193,1732422476559; all regions closed.
2024-11-24T04:29:31,903 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:31,903 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:31,903 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:31,903 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:31,904 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:31,906 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34465 is added to blk_1073741834_1010 (size=3066)
2024-11-24T04:29:31,907 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741834_1010 (size=3066)
2024-11-24T04:29:31,910 DEBUG [RS:0;fadb2fe0ba32:41193 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/oldWALs
2024-11-24T04:29:31,911 INFO  [RS:0;fadb2fe0ba32:41193 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog fadb2fe0ba32%2C41193%2C1732422476559.meta:.meta(num 1732422478384)
2024-11-24T04:29:31,911 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:31,911 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:31,911 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:31,912 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:31,912 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:31,913 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741847_1023 (size=12695)
2024-11-24T04:29:31,914 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34465 is added to blk_1073741847_1023 (size=12695)
2024-11-24T04:29:31,918 DEBUG [RS:0;fadb2fe0ba32:41193 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/oldWALs
2024-11-24T04:29:31,918 INFO  [RS:0;fadb2fe0ba32:41193 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog fadb2fe0ba32%2C41193%2C1732422476559:(num 1732422551247)
2024-11-24T04:29:31,918 DEBUG [RS:0;fadb2fe0ba32:41193 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T04:29:31,918 INFO  [RS:0;fadb2fe0ba32:41193 {}] regionserver.LeaseManager(133): Closed leases
2024-11-24T04:29:31,918 INFO  [RS:0;fadb2fe0ba32:41193 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-24T04:29:31,919 INFO  [RS:0;fadb2fe0ba32:41193 {}] hbase.ChoreService(370): Chore service for: regionserver/fadb2fe0ba32:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-24T04:29:31,919 INFO  [RS:0;fadb2fe0ba32:41193 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-24T04:29:31,919 INFO  [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-24T04:29:31,919 INFO  [RS:0;fadb2fe0ba32:41193 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41193
2024-11-24T04:29:31,924 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43529-0x101483b07a20000, quorum=127.0.0.1:57793, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-24T04:29:31,924 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41193-0x101483b07a20001, quorum=127.0.0.1:57793, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/fadb2fe0ba32,41193,1732422476559
2024-11-24T04:29:31,924 INFO  [RS:0;fadb2fe0ba32:41193 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-24T04:29:31,927 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [fadb2fe0ba32,41193,1732422476559]
2024-11-24T04:29:31,930 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/fadb2fe0ba32,41193,1732422476559 already deleted, retry=false
2024-11-24T04:29:31,930 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; fadb2fe0ba32,41193,1732422476559 expired; onlineServers=0
2024-11-24T04:29:31,930 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'fadb2fe0ba32,43529,1732422475859' *****
2024-11-24T04:29:31,930 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-24T04:29:31,930 INFO  [M:0;fadb2fe0ba32:43529 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-24T04:29:31,930 INFO  [M:0;fadb2fe0ba32:43529 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-24T04:29:31,930 DEBUG [M:0;fadb2fe0ba32:43529 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-24T04:29:31,930 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-24T04:29:31,930 DEBUG [M:0;fadb2fe0ba32:43529 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-24T04:29:31,930 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster-HFileCleaner.large.0-1732422477523 {}] cleaner.HFileCleaner(306): Exit Thread[master/fadb2fe0ba32:0:becomeActiveMaster-HFileCleaner.large.0-1732422477523,5,FailOnTimeoutGroup]
2024-11-24T04:29:31,930 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster-HFileCleaner.small.0-1732422477526 {}] cleaner.HFileCleaner(306): Exit Thread[master/fadb2fe0ba32:0:becomeActiveMaster-HFileCleaner.small.0-1732422477526,5,FailOnTimeoutGroup]
2024-11-24T04:29:31,931 INFO  [M:0;fadb2fe0ba32:43529 {}] hbase.ChoreService(370): Chore service for: master/fadb2fe0ba32:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-24T04:29:31,931 INFO  [M:0;fadb2fe0ba32:43529 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-24T04:29:31,931 DEBUG [M:0;fadb2fe0ba32:43529 {}] master.HMaster(1795): Stopping service threads
2024-11-24T04:29:31,931 INFO  [M:0;fadb2fe0ba32:43529 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-24T04:29:31,931 INFO  [M:0;fadb2fe0ba32:43529 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-24T04:29:31,931 INFO  [M:0;fadb2fe0ba32:43529 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-24T04:29:31,932 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-24T04:29:31,932 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43529-0x101483b07a20000, quorum=127.0.0.1:57793, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-24T04:29:31,932 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43529-0x101483b07a20000, quorum=127.0.0.1:57793, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:29:31,932 DEBUG [M:0;fadb2fe0ba32:43529 {}] zookeeper.ZKUtil(347): master:43529-0x101483b07a20000, quorum=127.0.0.1:57793, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-24T04:29:31,932 WARN  [M:0;fadb2fe0ba32:43529 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-24T04:29:31,933 INFO  [M:0;fadb2fe0ba32:43529 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/.lastflushedseqids
2024-11-24T04:29:31,945 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741854_1030 (size=130)
2024-11-24T04:29:31,945 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34465 is added to blk_1073741854_1030 (size=130)
2024-11-24T04:29:31,946 INFO  [M:0;fadb2fe0ba32:43529 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-24T04:29:31,946 INFO  [M:0;fadb2fe0ba32:43529 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-24T04:29:31,946 DEBUG [M:0;fadb2fe0ba32:43529 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-24T04:29:31,946 INFO  [M:0;fadb2fe0ba32:43529 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T04:29:31,946 DEBUG [M:0;fadb2fe0ba32:43529 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T04:29:31,946 DEBUG [M:0;fadb2fe0ba32:43529 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-24T04:29:31,946 DEBUG [M:0;fadb2fe0ba32:43529 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T04:29:31,947 INFO  [M:0;fadb2fe0ba32:43529 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.04 KB heapSize=29.21 KB
2024-11-24T04:29:31,965 DEBUG [M:0;fadb2fe0ba32:43529 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/06df006869f0434fb5629ab56968454f is 82, key is hbase:meta,,1/info:regioninfo/1732422478458/Put/seqid=0
2024-11-24T04:29:31,970 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34465 is added to blk_1073741855_1031 (size=5672)
2024-11-24T04:29:31,971 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741855_1031 (size=5672)
2024-11-24T04:29:31,971 INFO  [M:0;fadb2fe0ba32:43529 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/06df006869f0434fb5629ab56968454f
2024-11-24T04:29:31,993 DEBUG [M:0;fadb2fe0ba32:43529 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/14d8f04ace304930af51f29218613a69 is 767, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732422479244/Put/seqid=0
2024-11-24T04:29:31,998 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741856_1032 (size=6248)
2024-11-24T04:29:31,999 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34465 is added to blk_1073741856_1032 (size=6248)
2024-11-24T04:29:31,999 INFO  [M:0;fadb2fe0ba32:43529 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.43 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/14d8f04ace304930af51f29218613a69
2024-11-24T04:29:32,005 INFO  [M:0;fadb2fe0ba32:43529 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 14d8f04ace304930af51f29218613a69
2024-11-24T04:29:32,021 DEBUG [M:0;fadb2fe0ba32:43529 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2755f9a580fd4c408743b371ceccffdf is 69, key is fadb2fe0ba32,41193,1732422476559/rs:state/1732422477591/Put/seqid=0
2024-11-24T04:29:32,027 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41193-0x101483b07a20001, quorum=127.0.0.1:57793, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-24T04:29:32,027 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41193-0x101483b07a20001, quorum=127.0.0.1:57793, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-24T04:29:32,027 INFO  [RS:0;fadb2fe0ba32:41193 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-24T04:29:32,028 INFO  [RS:0;fadb2fe0ba32:41193 {}] regionserver.HRegionServer(1031): Exiting; stopping=fadb2fe0ba32,41193,1732422476559; zookeeper connection closed.
2024-11-24T04:29:32,028 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@666d12e6 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@666d12e6
2024-11-24T04:29:32,029 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-24T04:29:32,030 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34465 is added to blk_1073741857_1033 (size=5156)
2024-11-24T04:29:32,031 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741857_1033 (size=5156)
2024-11-24T04:29:32,031 INFO  [M:0;fadb2fe0ba32:43529 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2755f9a580fd4c408743b371ceccffdf
2024-11-24T04:29:32,056 DEBUG [M:0;fadb2fe0ba32:43529 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/85c20bacf9124167bce22f12752ae5ae is 52, key is load_balancer_on/state:d/1732422478735/Put/seqid=0
2024-11-24T04:29:32,072 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741858_1034 (size=5056)
2024-11-24T04:29:32,072 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34465 is added to blk_1073741858_1034 (size=5056)
2024-11-24T04:29:32,073 INFO  [M:0;fadb2fe0ba32:43529 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/85c20bacf9124167bce22f12752ae5ae
2024-11-24T04:29:32,080 DEBUG [M:0;fadb2fe0ba32:43529 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/06df006869f0434fb5629ab56968454f as hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/06df006869f0434fb5629ab56968454f
2024-11-24T04:29:32,086 INFO  [M:0;fadb2fe0ba32:43529 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/06df006869f0434fb5629ab56968454f, entries=8, sequenceid=59, filesize=5.5 K
2024-11-24T04:29:32,087 DEBUG [M:0;fadb2fe0ba32:43529 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/14d8f04ace304930af51f29218613a69 as hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/14d8f04ace304930af51f29218613a69
2024-11-24T04:29:32,093 INFO  [M:0;fadb2fe0ba32:43529 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 14d8f04ace304930af51f29218613a69
2024-11-24T04:29:32,093 INFO  [M:0;fadb2fe0ba32:43529 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/14d8f04ace304930af51f29218613a69, entries=6, sequenceid=59, filesize=6.1 K
2024-11-24T04:29:32,095 DEBUG [M:0;fadb2fe0ba32:43529 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2755f9a580fd4c408743b371ceccffdf as hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2755f9a580fd4c408743b371ceccffdf
2024-11-24T04:29:32,100 INFO  [M:0;fadb2fe0ba32:43529 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2755f9a580fd4c408743b371ceccffdf, entries=1, sequenceid=59, filesize=5.0 K
2024-11-24T04:29:32,101 DEBUG [M:0;fadb2fe0ba32:43529 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/85c20bacf9124167bce22f12752ae5ae as hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/85c20bacf9124167bce22f12752ae5ae
2024-11-24T04:29:32,107 INFO  [M:0;fadb2fe0ba32:43529 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/85c20bacf9124167bce22f12752ae5ae, entries=1, sequenceid=59, filesize=4.9 K
2024-11-24T04:29:32,109 INFO  [M:0;fadb2fe0ba32:43529 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.04 KB/23588, heapSize ~29.15 KB/29848, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 163ms, sequenceid=59, compaction requested=false
2024-11-24T04:29:32,110 INFO  [M:0;fadb2fe0ba32:43529 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T04:29:32,110 DEBUG [M:0;fadb2fe0ba32:43529 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732422571946Disabling compacts and flushes for region at 1732422571946Disabling writes for close at 1732422571946Obtaining lock to block concurrent updates at 1732422571947 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732422571947Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23588, getHeapSize=29848, getOffHeapSize=0, getCellsCount=70 at 1732422571947Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732422571948 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732422571948Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732422571964 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732422571964Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732422571977 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732422571992 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732422571993 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732422572005 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732422572020 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732422572021 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732422572038 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732422572056 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732422572056Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@396e54f7: reopening flushed file at 1732422572079 (+23 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@605b2f80: reopening flushed file at 1732422572086 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@16c0511c: reopening flushed file at 1732422572094 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5087fec2: reopening flushed file at 1732422572101 (+7 ms)Finished flush of dataSize ~23.04 KB/23588, heapSize ~29.15 KB/29848, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 163ms, sequenceid=59, compaction requested=false at 1732422572109 (+8 ms)Writing region close event to WAL at 1732422572110 (+1 ms)Closed at 1732422572110
2024-11-24T04:29:32,111 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:32,111 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:32,111 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:32,111 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:32,112 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:32,113 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34329 is added to blk_1073741830_1006 (size=27985)
2024-11-24T04:29:32,114 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34465 is added to blk_1073741830_1006 (size=27985)
2024-11-24T04:29:32,114 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-24T04:29:32,114 INFO  [M:0;fadb2fe0ba32:43529 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-24T04:29:32,115 INFO  [M:0;fadb2fe0ba32:43529 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43529
2024-11-24T04:29:32,115 INFO  [M:0;fadb2fe0ba32:43529 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-24T04:29:32,217 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43529-0x101483b07a20000, quorum=127.0.0.1:57793, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-24T04:29:32,217 INFO  [M:0;fadb2fe0ba32:43529 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-24T04:29:32,217 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43529-0x101483b07a20000, quorum=127.0.0.1:57793, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-24T04:29:32,222 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1bf97579{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T04:29:32,225 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@22b88bcb{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-24T04:29:32,225 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-24T04:29:32,225 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2d48d695{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-24T04:29:32,225 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@11effdcd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/03752be3-12ba-e59c-4567-8f2d22dd32ed/hadoop.log.dir/,STOPPED}
2024-11-24T04:29:32,228 WARN  [BP-1583467146-172.17.0.2-1732422473010 heartbeating to localhost/127.0.0.1:35431 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-24T04:29:32,228 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-24T04:29:32,228 WARN  [BP-1583467146-172.17.0.2-1732422473010 heartbeating to localhost/127.0.0.1:35431 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1583467146-172.17.0.2-1732422473010 (Datanode Uuid b765d968-3d03-437c-b7ca-4ee12294f80f) service to localhost/127.0.0.1:35431
2024-11-24T04:29:32,228 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-24T04:29:32,229 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/03752be3-12ba-e59c-4567-8f2d22dd32ed/cluster_6735412e-b74b-ee63-bfe9-fcdf79afd6b9/data/data3/current/BP-1583467146-172.17.0.2-1732422473010 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T04:29:32,229 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/03752be3-12ba-e59c-4567-8f2d22dd32ed/cluster_6735412e-b74b-ee63-bfe9-fcdf79afd6b9/data/data4/current/BP-1583467146-172.17.0.2-1732422473010 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T04:29:32,230 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-24T04:29:32,232 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7b07d1ba{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T04:29:32,233 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@43e0a762{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-24T04:29:32,233 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-24T04:29:32,233 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@371e191c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-24T04:29:32,233 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@28778f0f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/03752be3-12ba-e59c-4567-8f2d22dd32ed/hadoop.log.dir/,STOPPED}
2024-11-24T04:29:32,235 WARN  [BP-1583467146-172.17.0.2-1732422473010 heartbeating to localhost/127.0.0.1:35431 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-24T04:29:32,235 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-24T04:29:32,235 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-24T04:29:32,235 WARN  [BP-1583467146-172.17.0.2-1732422473010 heartbeating to localhost/127.0.0.1:35431 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1583467146-172.17.0.2-1732422473010 (Datanode Uuid 3d793753-93ca-4950-92cf-102a55538428) service to localhost/127.0.0.1:35431
2024-11-24T04:29:32,236 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/03752be3-12ba-e59c-4567-8f2d22dd32ed/cluster_6735412e-b74b-ee63-bfe9-fcdf79afd6b9/data/data1/current/BP-1583467146-172.17.0.2-1732422473010 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T04:29:32,236 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/03752be3-12ba-e59c-4567-8f2d22dd32ed/cluster_6735412e-b74b-ee63-bfe9-fcdf79afd6b9/data/data2/current/BP-1583467146-172.17.0.2-1732422473010 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T04:29:32,236 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-24T04:29:32,248 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@735fa16a{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-24T04:29:32,248 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6c26a5a3{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-24T04:29:32,248 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-24T04:29:32,249 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@70be1389{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-24T04:29:32,249 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@ddc8467{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/03752be3-12ba-e59c-4567-8f2d22dd32ed/hadoop.log.dir/,STOPPED}
2024-11-24T04:29:32,258 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-24T04:29:32,296 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-24T04:29:32,309 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=77 (was 12)
Potentially hanging thread: HMaster-EventLoopGroup-1-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Timer for 'HBase' metrics system
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: sync.0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: sync.1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: SnapshotHandlerChoreCleaner
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:35431
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:35431 from jenkins.hfs.0
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: sync.4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: HMaster-EventLoopGroup-1-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:35431
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Monitor thread for TaskMonitor
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:35431 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
	app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: SSL Certificates Store Monitor
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.Object.wait(Object.java:338)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:537)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: ForkJoinPool-2-worker-2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: nioEventLoopGroup-3-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:35431
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.0@localhost:35431
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-1
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-4-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RpcClient-timer-pool-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-2-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Async-Client-Retry-Timer-pool-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: region-location-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-5-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-3-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: nioEventLoopGroup-2-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-4-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: master/fadb2fe0ba32:0:becomeActiveMaster-MemStoreChunkPool Statistics
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: regionserver/fadb2fe0ba32:0.procedureResultReporter
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75)

Potentially hanging thread: sync.2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: master/fadb2fe0ba32:0:becomeActiveMaster-MemStoreChunkPool Statistics
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: weak-ref-cleaner-strictcontextstorage
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
	app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-3-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-4-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: GcTimeMonitor obsWindow = 60000, sleepInterval = 5000, maxGcTimePerc = 100
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.util.GcTimeMonitor.run(GcTimeMonitor.java:161)

Potentially hanging thread: HBase-Metrics2-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-5-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:35431 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:35431
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: SessionTracker
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163)

Potentially hanging thread: nioEventLoopGroup-2-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-5-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: ForkJoinPool-2-worker-3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: org.apache.hadoop.hdfs.PeerCache@1e598d20
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253)
	app//org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46)
	app//org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: sync.2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: Time-limited test.named-queue-events-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47)
	app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56)
	app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159)
	app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-1-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=402 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=148 (was 132) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7078 (was 6961) - AvailableMemoryMB LEAK? -
2024-11-24T04:29:32,316 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=78, OpenFileDescriptor=402, MaxFileDescriptor=1048576, SystemLoadAverage=148, ProcessCount=11, AvailableMemoryMB=7078
2024-11-24T04:29:32,316 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-24T04:29:32,317 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/03752be3-12ba-e59c-4567-8f2d22dd32ed/hadoop.log.dir so I do NOT create it in target/test-data/85930ece-2ad7-a1e7-d45a-6b50e739afc7
2024-11-24T04:29:32,317 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/03752be3-12ba-e59c-4567-8f2d22dd32ed/hadoop.tmp.dir so I do NOT create it in target/test-data/85930ece-2ad7-a1e7-d45a-6b50e739afc7
2024-11-24T04:29:32,317 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85930ece-2ad7-a1e7-d45a-6b50e739afc7/cluster_bd22959d-b714-a1e2-e9d7-576ceba76b35, deleteOnExit=true
2024-11-24T04:29:32,317 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-24T04:29:32,318 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85930ece-2ad7-a1e7-d45a-6b50e739afc7/test.cache.data in system properties and HBase conf
2024-11-24T04:29:32,318 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85930ece-2ad7-a1e7-d45a-6b50e739afc7/hadoop.tmp.dir in system properties and HBase conf
2024-11-24T04:29:32,318 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85930ece-2ad7-a1e7-d45a-6b50e739afc7/hadoop.log.dir in system properties and HBase conf
2024-11-24T04:29:32,318 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85930ece-2ad7-a1e7-d45a-6b50e739afc7/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-24T04:29:32,318 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85930ece-2ad7-a1e7-d45a-6b50e739afc7/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-24T04:29:32,318 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-24T04:29:32,319 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-24T04:29:32,319 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85930ece-2ad7-a1e7-d45a-6b50e739afc7/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-24T04:29:32,319 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85930ece-2ad7-a1e7-d45a-6b50e739afc7/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-24T04:29:32,319 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85930ece-2ad7-a1e7-d45a-6b50e739afc7/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-24T04:29:32,319 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85930ece-2ad7-a1e7-d45a-6b50e739afc7/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-24T04:29:32,320 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85930ece-2ad7-a1e7-d45a-6b50e739afc7/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-24T04:29:32,320 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85930ece-2ad7-a1e7-d45a-6b50e739afc7/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-24T04:29:32,320 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85930ece-2ad7-a1e7-d45a-6b50e739afc7/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-24T04:29:32,320 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85930ece-2ad7-a1e7-d45a-6b50e739afc7/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-24T04:29:32,320 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85930ece-2ad7-a1e7-d45a-6b50e739afc7/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-24T04:29:32,320 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85930ece-2ad7-a1e7-d45a-6b50e739afc7/nfs.dump.dir in system properties and HBase conf
2024-11-24T04:29:32,320 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85930ece-2ad7-a1e7-d45a-6b50e739afc7/java.io.tmpdir in system properties and HBase conf
2024-11-24T04:29:32,321 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85930ece-2ad7-a1e7-d45a-6b50e739afc7/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-24T04:29:32,321 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85930ece-2ad7-a1e7-d45a-6b50e739afc7/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-24T04:29:32,321 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85930ece-2ad7-a1e7-d45a-6b50e739afc7/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-24T04:29:32,338 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-24T04:29:32,439 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T04:29:32,447 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-24T04:29:32,459 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-24T04:29:32,459 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-24T04:29:32,459 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-24T04:29:32,460 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T04:29:32,461 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3197ca45{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85930ece-2ad7-a1e7-d45a-6b50e739afc7/hadoop.log.dir/,AVAILABLE}
2024-11-24T04:29:32,461 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@45e3157d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-24T04:29:32,593 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@511dc70f{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85930ece-2ad7-a1e7-d45a-6b50e739afc7/java.io.tmpdir/jetty-localhost-42133-hadoop-hdfs-3_4_1-tests_jar-_-any-5986562041469072629/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-24T04:29:32,594 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3e469283{HTTP/1.1, (http/1.1)}{localhost:42133}
2024-11-24T04:29:32,594 INFO  [Time-limited test {}] server.Server(415): Started @101655ms
2024-11-24T04:29:32,608 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-24T04:29:32,707 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T04:29:32,713 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-24T04:29:32,721 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-24T04:29:32,721 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-24T04:29:32,721 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-24T04:29:32,726 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4edee9ab{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85930ece-2ad7-a1e7-d45a-6b50e739afc7/hadoop.log.dir/,AVAILABLE}
2024-11-24T04:29:32,727 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@276f8783{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-24T04:29:32,859 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5d4bdc00{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85930ece-2ad7-a1e7-d45a-6b50e739afc7/java.io.tmpdir/jetty-localhost-44087-hadoop-hdfs-3_4_1-tests_jar-_-any-15494985317785009599/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T04:29:32,859 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@687b21ce{HTTP/1.1, (http/1.1)}{localhost:44087}
2024-11-24T04:29:32,860 INFO  [Time-limited test {}] server.Server(415): Started @101921ms
2024-11-24T04:29:32,861 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-24T04:29:32,920 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T04:29:32,925 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-24T04:29:32,928 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-24T04:29:32,928 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-24T04:29:32,929 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-24T04:29:32,929 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2b44e274{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85930ece-2ad7-a1e7-d45a-6b50e739afc7/hadoop.log.dir/,AVAILABLE}
2024-11-24T04:29:32,930 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@376d199b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-24T04:29:32,983 WARN  [Thread-438 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85930ece-2ad7-a1e7-d45a-6b50e739afc7/cluster_bd22959d-b714-a1e2-e9d7-576ceba76b35/data/data1/current/BP-1338711505-172.17.0.2-1732422572357/current, will proceed with Du for space computation calculation, 
2024-11-24T04:29:32,983 WARN  [Thread-439 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85930ece-2ad7-a1e7-d45a-6b50e739afc7/cluster_bd22959d-b714-a1e2-e9d7-576ceba76b35/data/data2/current/BP-1338711505-172.17.0.2-1732422572357/current, will proceed with Du for space computation calculation, 
2024-11-24T04:29:33,030 WARN  [Thread-417 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-24T04:29:33,034 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x20be684cc04c3857 with lease ID 0xc6187deb53a92b7e: Processing first storage report for DS-418ca411-9773-419d-97f7-914320074c66 from datanode DatanodeRegistration(127.0.0.1:45131, datanodeUuid=b0b5af9f-26ff-4e56-a0b4-117010d9b2b0, infoPort=39065, infoSecurePort=0, ipcPort=35807, storageInfo=lv=-57;cid=testClusterID;nsid=1435720703;c=1732422572357)
2024-11-24T04:29:33,035 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x20be684cc04c3857 with lease ID 0xc6187deb53a92b7e: from storage DS-418ca411-9773-419d-97f7-914320074c66 node DatanodeRegistration(127.0.0.1:45131, datanodeUuid=b0b5af9f-26ff-4e56-a0b4-117010d9b2b0, infoPort=39065, infoSecurePort=0, ipcPort=35807, storageInfo=lv=-57;cid=testClusterID;nsid=1435720703;c=1732422572357), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-24T04:29:33,035 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x20be684cc04c3857 with lease ID 0xc6187deb53a92b7e: Processing first storage report for DS-2d3c50ea-d9e4-45c6-991b-9f65590811ff from datanode DatanodeRegistration(127.0.0.1:45131, datanodeUuid=b0b5af9f-26ff-4e56-a0b4-117010d9b2b0, infoPort=39065, infoSecurePort=0, ipcPort=35807, storageInfo=lv=-57;cid=testClusterID;nsid=1435720703;c=1732422572357)
2024-11-24T04:29:33,035 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x20be684cc04c3857 with lease ID 0xc6187deb53a92b7e: from storage DS-2d3c50ea-d9e4-45c6-991b-9f65590811ff node DatanodeRegistration(127.0.0.1:45131, datanodeUuid=b0b5af9f-26ff-4e56-a0b4-117010d9b2b0, infoPort=39065, infoSecurePort=0, ipcPort=35807, storageInfo=lv=-57;cid=testClusterID;nsid=1435720703;c=1732422572357), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T04:29:33,076 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@824b6ae{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85930ece-2ad7-a1e7-d45a-6b50e739afc7/java.io.tmpdir/jetty-localhost-40365-hadoop-hdfs-3_4_1-tests_jar-_-any-8433318643176366849/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T04:29:33,076 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5d7e5c06{HTTP/1.1, (http/1.1)}{localhost:40365}
2024-11-24T04:29:33,076 INFO  [Time-limited test {}] server.Server(415): Started @102138ms
2024-11-24T04:29:33,078 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-24T04:29:33,211 WARN  [Thread-464 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85930ece-2ad7-a1e7-d45a-6b50e739afc7/cluster_bd22959d-b714-a1e2-e9d7-576ceba76b35/data/data3/current/BP-1338711505-172.17.0.2-1732422572357/current, will proceed with Du for space computation calculation, 
2024-11-24T04:29:33,214 WARN  [Thread-465 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85930ece-2ad7-a1e7-d45a-6b50e739afc7/cluster_bd22959d-b714-a1e2-e9d7-576ceba76b35/data/data4/current/BP-1338711505-172.17.0.2-1732422572357/current, will proceed with Du for space computation calculation, 
2024-11-24T04:29:33,258 WARN  [Thread-453 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-24T04:29:33,263 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2537332b2ce00db6 with lease ID 0xc6187deb53a92b7f: Processing first storage report for DS-2f2f766a-bf2e-4f30-875e-f0b9326f4193 from datanode DatanodeRegistration(127.0.0.1:45271, datanodeUuid=de4c9845-b483-4524-ad1a-5131a6b4bde0, infoPort=41999, infoSecurePort=0, ipcPort=35903, storageInfo=lv=-57;cid=testClusterID;nsid=1435720703;c=1732422572357)
2024-11-24T04:29:33,263 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2537332b2ce00db6 with lease ID 0xc6187deb53a92b7f: from storage DS-2f2f766a-bf2e-4f30-875e-f0b9326f4193 node DatanodeRegistration(127.0.0.1:45271, datanodeUuid=de4c9845-b483-4524-ad1a-5131a6b4bde0, infoPort=41999, infoSecurePort=0, ipcPort=35903, storageInfo=lv=-57;cid=testClusterID;nsid=1435720703;c=1732422572357), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T04:29:33,263 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2537332b2ce00db6 with lease ID 0xc6187deb53a92b7f: Processing first storage report for DS-552433c0-9cbd-4848-9ae8-bbb569693bbf from datanode DatanodeRegistration(127.0.0.1:45271, datanodeUuid=de4c9845-b483-4524-ad1a-5131a6b4bde0, infoPort=41999, infoSecurePort=0, ipcPort=35903, storageInfo=lv=-57;cid=testClusterID;nsid=1435720703;c=1732422572357)
2024-11-24T04:29:33,263 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2537332b2ce00db6 with lease ID 0xc6187deb53a92b7f: from storage DS-552433c0-9cbd-4848-9ae8-bbb569693bbf node DatanodeRegistration(127.0.0.1:45271, datanodeUuid=de4c9845-b483-4524-ad1a-5131a6b4bde0, infoPort=41999, infoSecurePort=0, ipcPort=35903, storageInfo=lv=-57;cid=testClusterID;nsid=1435720703;c=1732422572357), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T04:29:33,350 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85930ece-2ad7-a1e7-d45a-6b50e739afc7
2024-11-24T04:29:33,362 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85930ece-2ad7-a1e7-d45a-6b50e739afc7/cluster_bd22959d-b714-a1e2-e9d7-576ceba76b35/zookeeper_0, clientPort=49303, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85930ece-2ad7-a1e7-d45a-6b50e739afc7/cluster_bd22959d-b714-a1e2-e9d7-576ceba76b35/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85930ece-2ad7-a1e7-d45a-6b50e739afc7/cluster_bd22959d-b714-a1e2-e9d7-576ceba76b35/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-24T04:29:33,363 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=49303
2024-11-24T04:29:33,363 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T04:29:33,365 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T04:29:33,381 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45131 is added to blk_1073741825_1001 (size=7)
2024-11-24T04:29:33,381 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45271 is added to blk_1073741825_1001 (size=7)
2024-11-24T04:29:33,782 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:40093/user/jenkins/test-data/4b046d87-3de5-4fbe-6d1e-1604604a7218 with version=8
2024-11-24T04:29:33,782 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/hbase-staging
2024-11-24T04:29:33,785 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/fadb2fe0ba32:0 server-side Connection retries=45
2024-11-24T04:29:33,785 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-24T04:29:33,785 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-24T04:29:33,785 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-24T04:29:33,786 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-24T04:29:33,786 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-24T04:29:33,786 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-24T04:29:33,786 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-24T04:29:33,787 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39971
2024-11-24T04:29:33,789 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39971 connecting to ZooKeeper ensemble=127.0.0.1:49303
2024-11-24T04:29:33,797 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:399710x0, quorum=127.0.0.1:49303, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-24T04:29:33,798 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39971-0x101483c87bd0000 connected
2024-11-24T04:29:33,822 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T04:29:33,824 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T04:29:33,827 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39971-0x101483c87bd0000, quorum=127.0.0.1:49303, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-24T04:29:33,828 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:40093/user/jenkins/test-data/4b046d87-3de5-4fbe-6d1e-1604604a7218, hbase.cluster.distributed=false
2024-11-24T04:29:33,830 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39971-0x101483c87bd0000, quorum=127.0.0.1:49303, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-24T04:29:33,830 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39971
2024-11-24T04:29:33,831 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39971
2024-11-24T04:29:33,831 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39971
2024-11-24T04:29:33,833 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39971
2024-11-24T04:29:33,836 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39971
2024-11-24T04:29:33,858 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/fadb2fe0ba32:0 server-side Connection retries=45
2024-11-24T04:29:33,858 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-24T04:29:33,858 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-24T04:29:33,858 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-24T04:29:33,858 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-24T04:29:33,858 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-24T04:29:33,859 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-24T04:29:33,859 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-24T04:29:33,859 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38993
2024-11-24T04:29:33,861 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38993 connecting to ZooKeeper ensemble=127.0.0.1:49303
2024-11-24T04:29:33,862 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T04:29:33,864 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T04:29:33,870 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:389930x0, quorum=127.0.0.1:49303, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-24T04:29:33,871 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38993-0x101483c87bd0001 connected
2024-11-24T04:29:33,871 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38993-0x101483c87bd0001, quorum=127.0.0.1:49303, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-24T04:29:33,871 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-24T04:29:33,872 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-24T04:29:33,873 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38993-0x101483c87bd0001, quorum=127.0.0.1:49303, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-24T04:29:33,874 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38993-0x101483c87bd0001, quorum=127.0.0.1:49303, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-24T04:29:33,875 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38993
2024-11-24T04:29:33,875 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38993
2024-11-24T04:29:33,875 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38993
2024-11-24T04:29:33,875 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38993
2024-11-24T04:29:33,876 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38993
2024-11-24T04:29:33,890 DEBUG [M:0;fadb2fe0ba32:39971 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;fadb2fe0ba32:39971
2024-11-24T04:29:33,891 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/fadb2fe0ba32,39971,1732422573785
2024-11-24T04:29:33,893 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39971-0x101483c87bd0000, quorum=127.0.0.1:49303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-24T04:29:33,893 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x101483c87bd0001, quorum=127.0.0.1:49303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-24T04:29:33,894 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39971-0x101483c87bd0000, quorum=127.0.0.1:49303, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/fadb2fe0ba32,39971,1732422573785
2024-11-24T04:29:33,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x101483c87bd0001, quorum=127.0.0.1:49303, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-24T04:29:33,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x101483c87bd0001, quorum=127.0.0.1:49303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:29:33,896 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39971-0x101483c87bd0000, quorum=127.0.0.1:49303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:29:33,903 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39971-0x101483c87bd0000, quorum=127.0.0.1:49303, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-24T04:29:33,904 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/fadb2fe0ba32,39971,1732422573785 from backup master directory
2024-11-24T04:29:33,907 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x101483c87bd0001, quorum=127.0.0.1:49303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-24T04:29:33,907 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39971-0x101483c87bd0000, quorum=127.0.0.1:49303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/fadb2fe0ba32,39971,1732422573785
2024-11-24T04:29:33,907 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39971-0x101483c87bd0000, quorum=127.0.0.1:49303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-24T04:29:33,907 WARN  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-24T04:29:33,907 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=fadb2fe0ba32,39971,1732422573785
2024-11-24T04:29:33,918 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:40093/user/jenkins/test-data/4b046d87-3de5-4fbe-6d1e-1604604a7218/hbase.id] with ID: 8e29d456-2140-44e9-89d7-0fc648681c95
2024-11-24T04:29:33,918 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:40093/user/jenkins/test-data/4b046d87-3de5-4fbe-6d1e-1604604a7218/.tmp/hbase.id
2024-11-24T04:29:33,954 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45131 is added to blk_1073741826_1002 (size=42)
2024-11-24T04:29:33,955 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45271 is added to blk_1073741826_1002 (size=42)
2024-11-24T04:29:33,956 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:40093/user/jenkins/test-data/4b046d87-3de5-4fbe-6d1e-1604604a7218/.tmp/hbase.id]:[hdfs://localhost:40093/user/jenkins/test-data/4b046d87-3de5-4fbe-6d1e-1604604a7218/hbase.id]
2024-11-24T04:29:33,984 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T04:29:33,984 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-24T04:29:33,986 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms.
2024-11-24T04:29:33,989 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x101483c87bd0001, quorum=127.0.0.1:49303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:29:33,989 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39971-0x101483c87bd0000, quorum=127.0.0.1:49303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:29:34,019 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45131 is added to blk_1073741827_1003 (size=196)
2024-11-24T04:29:34,020 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45271 is added to blk_1073741827_1003 (size=196)
2024-11-24T04:29:34,022 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-24T04:29:34,023 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-24T04:29:34,024 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-24T04:29:34,057 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45271 is added to blk_1073741828_1004 (size=1189)
2024-11-24T04:29:34,057 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45131 is added to blk_1073741828_1004 (size=1189)
2024-11-24T04:29:34,059 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:40093/user/jenkins/test-data/4b046d87-3de5-4fbe-6d1e-1604604a7218/MasterData/data/master/store
2024-11-24T04:29:34,082 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45271 is added to blk_1073741829_1005 (size=34)
2024-11-24T04:29:34,083 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45131 is added to blk_1073741829_1005 (size=34)
2024-11-24T04:29:34,085 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T04:29:34,085 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-24T04:29:34,085 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T04:29:34,085 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T04:29:34,085 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-24T04:29:34,085 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T04:29:34,085 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T04:29:34,085 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732422574085Disabling compacts and flushes for region at 1732422574085Disabling writes for close at 1732422574085Writing region close event to WAL at 1732422574085Closed at 1732422574085
2024-11-24T04:29:34,087 WARN  [master/fadb2fe0ba32:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:40093/user/jenkins/test-data/4b046d87-3de5-4fbe-6d1e-1604604a7218/MasterData/data/master/store/.initializing
2024-11-24T04:29:34,087 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:40093/user/jenkins/test-data/4b046d87-3de5-4fbe-6d1e-1604604a7218/MasterData/WALs/fadb2fe0ba32,39971,1732422573785
2024-11-24T04:29:34,092 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fadb2fe0ba32%2C39971%2C1732422573785, suffix=, logDir=hdfs://localhost:40093/user/jenkins/test-data/4b046d87-3de5-4fbe-6d1e-1604604a7218/MasterData/WALs/fadb2fe0ba32,39971,1732422573785, archiveDir=hdfs://localhost:40093/user/jenkins/test-data/4b046d87-3de5-4fbe-6d1e-1604604a7218/MasterData/oldWALs, maxLogs=10
2024-11-24T04:29:34,093 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor fadb2fe0ba32%2C39971%2C1732422573785.1732422574093
2024-11-24T04:29:34,122 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4b046d87-3de5-4fbe-6d1e-1604604a7218/MasterData/WALs/fadb2fe0ba32,39971,1732422573785/fadb2fe0ba32%2C39971%2C1732422573785.1732422574093
2024-11-24T04:29:34,129 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41999:41999),(127.0.0.1/127.0.0.1:39065:39065)]
2024-11-24T04:29:34,130 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-24T04:29:34,130 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T04:29:34,130 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T04:29:34,130 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T04:29:34,136 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-24T04:29:34,138 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-24T04:29:34,138 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:29:34,139 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T04:29:34,139 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-24T04:29:34,141 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-24T04:29:34,141 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:29:34,141 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-24T04:29:34,141 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-24T04:29:34,144 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-24T04:29:34,144 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:29:34,145 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-24T04:29:34,145 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-24T04:29:34,146 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-24T04:29:34,146 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:29:34,147 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-24T04:29:34,147 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T04:29:34,148 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40093/user/jenkins/test-data/4b046d87-3de5-4fbe-6d1e-1604604a7218/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-24T04:29:34,148 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40093/user/jenkins/test-data/4b046d87-3de5-4fbe-6d1e-1604604a7218/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-24T04:29:34,149 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T04:29:34,149 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T04:29:34,150 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-24T04:29:34,152 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T04:29:34,157 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40093/user/jenkins/test-data/4b046d87-3de5-4fbe-6d1e-1604604a7218/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-24T04:29:34,157 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=769069, jitterRate=-0.022078588604927063}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-24T04:29:34,159 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1732422574131Initializing all the Stores at 1732422574133 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732422574133Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422574136 (+3 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422574136Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422574136Cleaning up temporary data from old regions at 1732422574150 (+14 ms)Region opened successfully at 1732422574159 (+9 ms)
2024-11-24T04:29:34,165 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-24T04:29:34,173 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5860e957, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fadb2fe0ba32/172.17.0.2:0
2024-11-24T04:29:34,175 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-24T04:29:34,175 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-24T04:29:34,175 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-24T04:29:34,175 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-24T04:29:34,176 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-24T04:29:34,176 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-24T04:29:34,177 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-24T04:29:34,179 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-24T04:29:34,180 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39971-0x101483c87bd0000, quorum=127.0.0.1:49303, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-24T04:29:34,181 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-24T04:29:34,182 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-24T04:29:34,182 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39971-0x101483c87bd0000, quorum=127.0.0.1:49303, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-24T04:29:34,184 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-24T04:29:34,184 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-24T04:29:34,185 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39971-0x101483c87bd0000, quorum=127.0.0.1:49303, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-24T04:29:34,188 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-24T04:29:34,188 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39971-0x101483c87bd0000, quorum=127.0.0.1:49303, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-24T04:29:34,189 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-24T04:29:34,192 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39971-0x101483c87bd0000, quorum=127.0.0.1:49303, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-24T04:29:34,193 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-24T04:29:34,195 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39971-0x101483c87bd0000, quorum=127.0.0.1:49303, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-24T04:29:34,195 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x101483c87bd0001, quorum=127.0.0.1:49303, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-24T04:29:34,195 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39971-0x101483c87bd0000, quorum=127.0.0.1:49303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:29:34,195 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x101483c87bd0001, quorum=127.0.0.1:49303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:29:34,196 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=fadb2fe0ba32,39971,1732422573785, sessionid=0x101483c87bd0000, setting cluster-up flag (Was=false)
2024-11-24T04:29:34,200 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39971-0x101483c87bd0000, quorum=127.0.0.1:49303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:29:34,200 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x101483c87bd0001, quorum=127.0.0.1:49303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:29:34,209 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-24T04:29:34,210 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=fadb2fe0ba32,39971,1732422573785
2024-11-24T04:29:34,213 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39971-0x101483c87bd0000, quorum=127.0.0.1:49303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:29:34,214 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x101483c87bd0001, quorum=127.0.0.1:49303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:29:34,220 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-24T04:29:34,222 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=fadb2fe0ba32,39971,1732422573785
2024-11-24T04:29:34,223 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:40093/user/jenkins/test-data/4b046d87-3de5-4fbe-6d1e-1604604a7218/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-24T04:29:34,225 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-24T04:29:34,225 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-24T04:29:34,226 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-24T04:29:34,226 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: fadb2fe0ba32,39971,1732422573785
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-24T04:29:34,227 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/fadb2fe0ba32:0, corePoolSize=5, maxPoolSize=5
2024-11-24T04:29:34,227 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/fadb2fe0ba32:0, corePoolSize=5, maxPoolSize=5
2024-11-24T04:29:34,228 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/fadb2fe0ba32:0, corePoolSize=5, maxPoolSize=5
2024-11-24T04:29:34,228 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/fadb2fe0ba32:0, corePoolSize=5, maxPoolSize=5
2024-11-24T04:29:34,228 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/fadb2fe0ba32:0, corePoolSize=10, maxPoolSize=10
2024-11-24T04:29:34,228 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:29:34,228 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/fadb2fe0ba32:0, corePoolSize=2, maxPoolSize=2
2024-11-24T04:29:34,228 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:29:34,229 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732422604229
2024-11-24T04:29:34,229 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-24T04:29:34,229 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-24T04:29:34,229 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-24T04:29:34,229 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-24T04:29:34,229 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-24T04:29:34,229 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-24T04:29:34,230 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-24T04:29:34,230 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-24T04:29:34,230 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-24T04:29:34,230 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-24T04:29:34,230 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-24T04:29:34,230 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-24T04:29:34,230 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-24T04:29:34,230 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-24T04:29:34,231 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/fadb2fe0ba32:0:becomeActiveMaster-HFileCleaner.large.0-1732422574231,5,FailOnTimeoutGroup]
2024-11-24T04:29:34,231 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/fadb2fe0ba32:0:becomeActiveMaster-HFileCleaner.small.0-1732422574231,5,FailOnTimeoutGroup]
2024-11-24T04:29:34,231 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-24T04:29:34,231 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:29:34,231 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-24T04:29:34,231 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-24T04:29:34,231 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-24T04:29:34,231 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-24T04:29:34,240 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45131 is added to blk_1073741831_1007 (size=1321)
2024-11-24T04:29:34,241 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45271 is added to blk_1073741831_1007 (size=1321)
2024-11-24T04:29:34,241 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:40093/user/jenkins/test-data/4b046d87-3de5-4fbe-6d1e-1604604a7218/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-24T04:29:34,242 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:40093/user/jenkins/test-data/4b046d87-3de5-4fbe-6d1e-1604604a7218
2024-11-24T04:29:34,251 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45131 is added to blk_1073741832_1008 (size=32)
2024-11-24T04:29:34,251 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45271 is added to blk_1073741832_1008 (size=32)
2024-11-24T04:29:34,252 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T04:29:34,254 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-24T04:29:34,258 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-24T04:29:34,258 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:29:34,259 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T04:29:34,259 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-24T04:29:34,261 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-24T04:29:34,261 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:29:34,261 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T04:29:34,261 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-24T04:29:34,263 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-24T04:29:34,263 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:29:34,264 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T04:29:34,264 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-24T04:29:34,266 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-24T04:29:34,266 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:29:34,266 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T04:29:34,267 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-24T04:29:34,268 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40093/user/jenkins/test-data/4b046d87-3de5-4fbe-6d1e-1604604a7218/data/hbase/meta/1588230740
2024-11-24T04:29:34,268 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40093/user/jenkins/test-data/4b046d87-3de5-4fbe-6d1e-1604604a7218/data/hbase/meta/1588230740
2024-11-24T04:29:34,270 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-24T04:29:34,270 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-24T04:29:34,270 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-24T04:29:34,272 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-24T04:29:34,275 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40093/user/jenkins/test-data/4b046d87-3de5-4fbe-6d1e-1604604a7218/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-24T04:29:34,276 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=817877, jitterRate=0.03998561203479767}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-24T04:29:34,277 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1732422574252Initializing all the Stores at 1732422574253 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732422574253Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732422574253Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422574253Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732422574253Cleaning up temporary data from old regions at 1732422574270 (+17 ms)Region opened successfully at 1732422574277 (+7 ms)
2024-11-24T04:29:34,277 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-24T04:29:34,277 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-24T04:29:34,277 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-24T04:29:34,277 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-24T04:29:34,277 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-24T04:29:34,279 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-24T04:29:34,279 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732422574277Disabling compacts and flushes for region at 1732422574277Disabling writes for close at 1732422574277Writing region close event to WAL at 1732422574279 (+2 ms)Closed at 1732422574279
2024-11-24T04:29:34,280 INFO  [RS:0;fadb2fe0ba32:38993 {}] regionserver.HRegionServer(746): ClusterId : 8e29d456-2140-44e9-89d7-0fc648681c95
2024-11-24T04:29:34,281 DEBUG [RS:0;fadb2fe0ba32:38993 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-24T04:29:34,281 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-24T04:29:34,281 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-24T04:29:34,281 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-24T04:29:34,283 INFO  [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-24T04:29:34,284 DEBUG [RS:0;fadb2fe0ba32:38993 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-24T04:29:34,284 DEBUG [RS:0;fadb2fe0ba32:38993 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-24T04:29:34,284 INFO  [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-24T04:29:34,287 DEBUG [RS:0;fadb2fe0ba32:38993 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-24T04:29:34,288 DEBUG [RS:0;fadb2fe0ba32:38993 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c12a66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fadb2fe0ba32/172.17.0.2:0
2024-11-24T04:29:34,301 DEBUG [RS:0;fadb2fe0ba32:38993 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;fadb2fe0ba32:38993
2024-11-24T04:29:34,301 INFO  [RS:0;fadb2fe0ba32:38993 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-24T04:29:34,301 INFO  [RS:0;fadb2fe0ba32:38993 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-24T04:29:34,301 DEBUG [RS:0;fadb2fe0ba32:38993 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-24T04:29:34,302 INFO  [RS:0;fadb2fe0ba32:38993 {}] regionserver.HRegionServer(2659): reportForDuty to master=fadb2fe0ba32,39971,1732422573785 with port=38993, startcode=1732422573858
2024-11-24T04:29:34,302 DEBUG [RS:0;fadb2fe0ba32:38993 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-24T04:29:34,305 INFO  [HMaster-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46703, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService
2024-11-24T04:29:34,306 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39971 {}] master.ServerManager(363): Checking decommissioned status of RegionServer fadb2fe0ba32,38993,1732422573858
2024-11-24T04:29:34,306 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39971 {}] master.ServerManager(517): Registering regionserver=fadb2fe0ba32,38993,1732422573858
2024-11-24T04:29:34,308 DEBUG [RS:0;fadb2fe0ba32:38993 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40093/user/jenkins/test-data/4b046d87-3de5-4fbe-6d1e-1604604a7218
2024-11-24T04:29:34,308 DEBUG [RS:0;fadb2fe0ba32:38993 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40093
2024-11-24T04:29:34,309 DEBUG [RS:0;fadb2fe0ba32:38993 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-24T04:29:34,311 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39971-0x101483c87bd0000, quorum=127.0.0.1:49303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-24T04:29:34,312 DEBUG [RS:0;fadb2fe0ba32:38993 {}] zookeeper.ZKUtil(111): regionserver:38993-0x101483c87bd0001, quorum=127.0.0.1:49303, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/fadb2fe0ba32,38993,1732422573858
2024-11-24T04:29:34,312 WARN  [RS:0;fadb2fe0ba32:38993 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-24T04:29:34,312 INFO  [RS:0;fadb2fe0ba32:38993 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-24T04:29:34,312 DEBUG [RS:0;fadb2fe0ba32:38993 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40093/user/jenkins/test-data/4b046d87-3de5-4fbe-6d1e-1604604a7218/WALs/fadb2fe0ba32,38993,1732422573858
2024-11-24T04:29:34,312 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [fadb2fe0ba32,38993,1732422573858]
2024-11-24T04:29:34,316 INFO  [RS:0;fadb2fe0ba32:38993 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-24T04:29:34,319 INFO  [RS:0;fadb2fe0ba32:38993 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-24T04:29:34,319 INFO  [RS:0;fadb2fe0ba32:38993 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-24T04:29:34,319 INFO  [RS:0;fadb2fe0ba32:38993 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-24T04:29:34,320 INFO  [RS:0;fadb2fe0ba32:38993 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-24T04:29:34,321 INFO  [RS:0;fadb2fe0ba32:38993 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-24T04:29:34,321 INFO  [RS:0;fadb2fe0ba32:38993 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-24T04:29:34,321 DEBUG [RS:0;fadb2fe0ba32:38993 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:29:34,321 DEBUG [RS:0;fadb2fe0ba32:38993 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:29:34,321 DEBUG [RS:0;fadb2fe0ba32:38993 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:29:34,321 DEBUG [RS:0;fadb2fe0ba32:38993 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:29:34,321 DEBUG [RS:0;fadb2fe0ba32:38993 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:29:34,321 DEBUG [RS:0;fadb2fe0ba32:38993 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/fadb2fe0ba32:0, corePoolSize=2, maxPoolSize=2
2024-11-24T04:29:34,321 DEBUG [RS:0;fadb2fe0ba32:38993 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:29:34,321 DEBUG [RS:0;fadb2fe0ba32:38993 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:29:34,321 DEBUG [RS:0;fadb2fe0ba32:38993 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:29:34,321 DEBUG [RS:0;fadb2fe0ba32:38993 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:29:34,322 DEBUG [RS:0;fadb2fe0ba32:38993 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:29:34,322 DEBUG [RS:0;fadb2fe0ba32:38993 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:29:34,322 DEBUG [RS:0;fadb2fe0ba32:38993 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/fadb2fe0ba32:0, corePoolSize=3, maxPoolSize=3
2024-11-24T04:29:34,322 DEBUG [RS:0;fadb2fe0ba32:38993 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/fadb2fe0ba32:0, corePoolSize=3, maxPoolSize=3
2024-11-24T04:29:34,324 INFO  [RS:0;fadb2fe0ba32:38993 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-24T04:29:34,324 INFO  [RS:0;fadb2fe0ba32:38993 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-24T04:29:34,324 INFO  [RS:0;fadb2fe0ba32:38993 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-24T04:29:34,324 INFO  [RS:0;fadb2fe0ba32:38993 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-24T04:29:34,324 INFO  [RS:0;fadb2fe0ba32:38993 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-24T04:29:34,324 INFO  [RS:0;fadb2fe0ba32:38993 {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,38993,1732422573858-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-24T04:29:34,348 INFO  [RS:0;fadb2fe0ba32:38993 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-24T04:29:34,348 INFO  [RS:0;fadb2fe0ba32:38993 {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,38993,1732422573858-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-24T04:29:34,348 INFO  [RS:0;fadb2fe0ba32:38993 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T04:29:34,348 INFO  [RS:0;fadb2fe0ba32:38993 {}] regionserver.Replication(171): fadb2fe0ba32,38993,1732422573858 started
2024-11-24T04:29:34,371 INFO  [RS:0;fadb2fe0ba32:38993 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T04:29:34,371 INFO  [RS:0;fadb2fe0ba32:38993 {}] regionserver.HRegionServer(1482): Serving as fadb2fe0ba32,38993,1732422573858, RpcServer on fadb2fe0ba32/172.17.0.2:38993, sessionid=0x101483c87bd0001
2024-11-24T04:29:34,372 DEBUG [RS:0;fadb2fe0ba32:38993 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-24T04:29:34,372 DEBUG [RS:0;fadb2fe0ba32:38993 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager fadb2fe0ba32,38993,1732422573858
2024-11-24T04:29:34,372 DEBUG [RS:0;fadb2fe0ba32:38993 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fadb2fe0ba32,38993,1732422573858'
2024-11-24T04:29:34,372 DEBUG [RS:0;fadb2fe0ba32:38993 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-24T04:29:34,373 DEBUG [RS:0;fadb2fe0ba32:38993 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-24T04:29:34,373 DEBUG [RS:0;fadb2fe0ba32:38993 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-24T04:29:34,373 DEBUG [RS:0;fadb2fe0ba32:38993 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-24T04:29:34,373 DEBUG [RS:0;fadb2fe0ba32:38993 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager fadb2fe0ba32,38993,1732422573858
2024-11-24T04:29:34,373 DEBUG [RS:0;fadb2fe0ba32:38993 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fadb2fe0ba32,38993,1732422573858'
2024-11-24T04:29:34,373 DEBUG [RS:0;fadb2fe0ba32:38993 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-24T04:29:34,374 DEBUG [RS:0;fadb2fe0ba32:38993 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-24T04:29:34,375 DEBUG [RS:0;fadb2fe0ba32:38993 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-24T04:29:34,375 INFO  [RS:0;fadb2fe0ba32:38993 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-24T04:29:34,375 INFO  [RS:0;fadb2fe0ba32:38993 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-24T04:29:34,435 WARN  [fadb2fe0ba32:39971 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-11-24T04:29:34,477 INFO  [RS:0;fadb2fe0ba32:38993 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fadb2fe0ba32%2C38993%2C1732422573858, suffix=, logDir=hdfs://localhost:40093/user/jenkins/test-data/4b046d87-3de5-4fbe-6d1e-1604604a7218/WALs/fadb2fe0ba32,38993,1732422573858, archiveDir=hdfs://localhost:40093/user/jenkins/test-data/4b046d87-3de5-4fbe-6d1e-1604604a7218/oldWALs, maxLogs=32
2024-11-24T04:29:34,479 INFO  [RS:0;fadb2fe0ba32:38993 {}] monitor.StreamSlowMonitor(122): New stream slow monitor fadb2fe0ba32%2C38993%2C1732422573858.1732422574479
2024-11-24T04:29:34,486 INFO  [RS:0;fadb2fe0ba32:38993 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4b046d87-3de5-4fbe-6d1e-1604604a7218/WALs/fadb2fe0ba32,38993,1732422573858/fadb2fe0ba32%2C38993%2C1732422573858.1732422574479
2024-11-24T04:29:34,487 DEBUG [RS:0;fadb2fe0ba32:38993 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39065:39065),(127.0.0.1/127.0.0.1:41999:41999)]
2024-11-24T04:29:34,685 DEBUG [fadb2fe0ba32:39971 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-24T04:29:34,686 INFO  [PEWorker-2 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=fadb2fe0ba32,38993,1732422573858
2024-11-24T04:29:34,688 INFO  [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fadb2fe0ba32,38993,1732422573858, state=OPENING
2024-11-24T04:29:34,689 DEBUG [PEWorker-2 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-24T04:29:34,691 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x101483c87bd0001, quorum=127.0.0.1:49303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:29:34,691 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39971-0x101483c87bd0000, quorum=127.0.0.1:49303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:29:34,692 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-24T04:29:34,692 DEBUG [PEWorker-2 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-24T04:29:34,692 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-24T04:29:34,692 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=fadb2fe0ba32,38993,1732422573858}]
2024-11-24T04:29:34,846 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-24T04:29:34,849 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53211, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-24T04:29:34,853 INFO  [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-24T04:29:34,853 INFO  [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-24T04:29:34,856 INFO  [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fadb2fe0ba32%2C38993%2C1732422573858.meta, suffix=.meta, logDir=hdfs://localhost:40093/user/jenkins/test-data/4b046d87-3de5-4fbe-6d1e-1604604a7218/WALs/fadb2fe0ba32,38993,1732422573858, archiveDir=hdfs://localhost:40093/user/jenkins/test-data/4b046d87-3de5-4fbe-6d1e-1604604a7218/oldWALs, maxLogs=32
2024-11-24T04:29:34,858 INFO  [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor fadb2fe0ba32%2C38993%2C1732422573858.meta.1732422574858.meta
2024-11-24T04:29:34,865 INFO  [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4b046d87-3de5-4fbe-6d1e-1604604a7218/WALs/fadb2fe0ba32,38993,1732422573858/fadb2fe0ba32%2C38993%2C1732422573858.meta.1732422574858.meta
2024-11-24T04:29:34,866 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39065:39065),(127.0.0.1/127.0.0.1:41999:41999)]
2024-11-24T04:29:34,869 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-24T04:29:34,869 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-24T04:29:34,869 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-24T04:29:34,870 INFO  [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-24T04:29:34,870 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-24T04:29:34,870 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T04:29:34,870 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-24T04:29:34,870 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-24T04:29:34,872 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-24T04:29:34,873 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-24T04:29:34,873 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:29:34,874 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T04:29:34,874 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-24T04:29:34,875 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-24T04:29:34,875 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:29:34,876 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T04:29:34,876 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-24T04:29:34,877 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-24T04:29:34,877 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:29:34,878 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T04:29:34,878 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-24T04:29:34,878 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-24T04:29:34,879 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:29:34,879 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T04:29:34,879 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-24T04:29:34,880 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40093/user/jenkins/test-data/4b046d87-3de5-4fbe-6d1e-1604604a7218/data/hbase/meta/1588230740
2024-11-24T04:29:34,882 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40093/user/jenkins/test-data/4b046d87-3de5-4fbe-6d1e-1604604a7218/data/hbase/meta/1588230740
2024-11-24T04:29:34,883 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-24T04:29:34,883 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-24T04:29:34,884 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-24T04:29:34,886 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-24T04:29:34,887 INFO  [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=696084, jitterRate=-0.11488468945026398}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-24T04:29:34,887 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-24T04:29:34,889 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1732422574870Writing region info on filesystem at 1732422574870Initializing all the Stores at 1732422574872 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732422574872Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732422574872Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422574872Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732422574872Cleaning up temporary data from old regions at 1732422574884 (+12 ms)Running coprocessor post-open hooks at 1732422574887 (+3 ms)Region opened successfully at 1732422574889 (+2 ms)
2024-11-24T04:29:34,890 INFO  [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732422574846
2024-11-24T04:29:34,893 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-24T04:29:34,893 INFO  [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-24T04:29:34,895 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=fadb2fe0ba32,38993,1732422573858
2024-11-24T04:29:34,896 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fadb2fe0ba32,38993,1732422573858, state=OPEN
2024-11-24T04:29:34,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x101483c87bd0001, quorum=127.0.0.1:49303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-24T04:29:34,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39971-0x101483c87bd0000, quorum=127.0.0.1:49303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-24T04:29:34,904 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=fadb2fe0ba32,38993,1732422573858
2024-11-24T04:29:34,904 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-24T04:29:34,904 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-24T04:29:34,907 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-24T04:29:34,907 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fadb2fe0ba32,38993,1732422573858 in 212 msec
2024-11-24T04:29:34,910 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-24T04:29:34,910 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 626 msec
2024-11-24T04:29:34,911 DEBUG [PEWorker-3 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-24T04:29:34,911 INFO  [PEWorker-3 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-24T04:29:34,913 DEBUG [PEWorker-3 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-24T04:29:34,913 DEBUG [PEWorker-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fadb2fe0ba32,38993,1732422573858, seqNum=-1]
2024-11-24T04:29:34,913 DEBUG [PEWorker-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-24T04:29:34,915 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36149, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-24T04:29:34,922 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 696 msec
2024-11-24T04:29:34,922 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732422574922, completionTime=-1
2024-11-24T04:29:34,922 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-24T04:29:34,922 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-24T04:29:34,924 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-24T04:29:34,924 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732422634924
2024-11-24T04:29:34,924 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732422694924
2024-11-24T04:29:34,924 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec
2024-11-24T04:29:34,924 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,39971,1732422573785-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-24T04:29:34,925 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,39971,1732422573785-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T04:29:34,925 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,39971,1732422573785-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T04:29:34,925 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-fadb2fe0ba32:39971, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T04:29:34,925 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-24T04:29:34,925 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-24T04:29:34,927 DEBUG [master/fadb2fe0ba32:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-24T04:29:34,929 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.022sec
2024-11-24T04:29:34,929 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-24T04:29:34,929 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-24T04:29:34,930 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-24T04:29:34,930 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-24T04:29:34,930 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-24T04:29:34,930 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,39971,1732422573785-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-24T04:29:34,930 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,39971,1732422573785-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-24T04:29:34,932 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-24T04:29:34,932 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-24T04:29:34,932 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,39971,1732422573785-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T04:29:34,980 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ffb657f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-24T04:29:34,980 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fadb2fe0ba32,39971,-1 for getting cluster id
2024-11-24T04:29:34,981 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-24T04:29:34,983 DEBUG [HMaster-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8e29d456-2140-44e9-89d7-0fc648681c95'
2024-11-24T04:29:34,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-24T04:29:34,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8e29d456-2140-44e9-89d7-0fc648681c95"

2024-11-24T04:29:34,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@564619b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-24T04:29:34,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fadb2fe0ba32,39971,-1]
2024-11-24T04:29:34,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-24T04:29:34,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T04:29:34,986 INFO  [HMaster-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34406, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-24T04:29:34,987 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e5d9cb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-24T04:29:34,987 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-24T04:29:34,989 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fadb2fe0ba32,38993,1732422573858, seqNum=-1]
2024-11-24T04:29:34,989 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-24T04:29:34,991 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50496, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-24T04:29:34,993 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=fadb2fe0ba32,39971,1732422573785
2024-11-24T04:29:34,993 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T04:29:34,996 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-24T04:29:34,997 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-24T04:29:34,997 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-24T04:29:34,997 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at org.apache.hadoop.hbase.regionserver.wal.TestLogRolling.testLogRollOnDatanodeDeath(TestLogRolling.java:201)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-24T04:29:34,997 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T04:29:34,997 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T04:29:34,997 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-24T04:29:34,997 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-24T04:29:34,997 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1644424511, stopped=false
2024-11-24T04:29:34,997 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=fadb2fe0ba32,39971,1732422573785
2024-11-24T04:29:34,999 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39971-0x101483c87bd0000, quorum=127.0.0.1:49303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-24T04:29:34,999 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x101483c87bd0001, quorum=127.0.0.1:49303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-24T04:29:34,999 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39971-0x101483c87bd0000, quorum=127.0.0.1:49303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:29:34,999 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x101483c87bd0001, quorum=127.0.0.1:49303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:29:34,999 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-24T04:29:35,000 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-24T04:29:35,000 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at org.apache.hadoop.hbase.regionserver.wal.TestLogRolling.testLogRollOnDatanodeDeath(TestLogRolling.java:201)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-24T04:29:35,000 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T04:29:35,000 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38993-0x101483c87bd0001, quorum=127.0.0.1:49303, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-24T04:29:35,000 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'fadb2fe0ba32,38993,1732422573858' *****
2024-11-24T04:29:35,001 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-24T04:29:35,001 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39971-0x101483c87bd0000, quorum=127.0.0.1:49303, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-24T04:29:35,001 INFO  [RS:0;fadb2fe0ba32:38993 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-24T04:29:35,001 INFO  [RS:0;fadb2fe0ba32:38993 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-24T04:29:35,001 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-24T04:29:35,001 INFO  [RS:0;fadb2fe0ba32:38993 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-24T04:29:35,001 INFO  [RS:0;fadb2fe0ba32:38993 {}] regionserver.HRegionServer(959): stopping server fadb2fe0ba32,38993,1732422573858
2024-11-24T04:29:35,001 INFO  [RS:0;fadb2fe0ba32:38993 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-24T04:29:35,001 INFO  [RS:0;fadb2fe0ba32:38993 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;fadb2fe0ba32:38993.
2024-11-24T04:29:35,001 DEBUG [RS:0;fadb2fe0ba32:38993 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-24T04:29:35,001 DEBUG [RS:0;fadb2fe0ba32:38993 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T04:29:35,002 INFO  [RS:0;fadb2fe0ba32:38993 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-24T04:29:35,002 INFO  [RS:0;fadb2fe0ba32:38993 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-24T04:29:35,002 INFO  [RS:0;fadb2fe0ba32:38993 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-24T04:29:35,002 INFO  [RS:0;fadb2fe0ba32:38993 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-24T04:29:35,002 INFO  [RS:0;fadb2fe0ba32:38993 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close
2024-11-24T04:29:35,002 DEBUG [RS:0;fadb2fe0ba32:38993 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740}
2024-11-24T04:29:35,002 DEBUG [RS:0;fadb2fe0ba32:38993 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-24T04:29:35,002 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-24T04:29:35,002 INFO  [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-24T04:29:35,002 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-24T04:29:35,003 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-24T04:29:35,003 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-24T04:29:35,003 INFO  [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=74 B heapSize=1.22 KB
2024-11-24T04:29:35,029 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40093/user/jenkins/test-data/4b046d87-3de5-4fbe-6d1e-1604604a7218/data/hbase/meta/1588230740/.tmp/ns/af19fb446bae466b91e6dc0aa1a11ef6 is 43, key is default/ns:d/1732422574915/Put/seqid=0
2024-11-24T04:29:35,035 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45271 is added to blk_1073741835_1011 (size=5153)
2024-11-24T04:29:35,035 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45131 is added to blk_1073741835_1011 (size=5153)
2024-11-24T04:29:35,036 INFO  [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:40093/user/jenkins/test-data/4b046d87-3de5-4fbe-6d1e-1604604a7218/data/hbase/meta/1588230740/.tmp/ns/af19fb446bae466b91e6dc0aa1a11ef6
2024-11-24T04:29:35,044 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40093/user/jenkins/test-data/4b046d87-3de5-4fbe-6d1e-1604604a7218/data/hbase/meta/1588230740/.tmp/ns/af19fb446bae466b91e6dc0aa1a11ef6 as hdfs://localhost:40093/user/jenkins/test-data/4b046d87-3de5-4fbe-6d1e-1604604a7218/data/hbase/meta/1588230740/ns/af19fb446bae466b91e6dc0aa1a11ef6
2024-11-24T04:29:35,052 INFO  [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40093/user/jenkins/test-data/4b046d87-3de5-4fbe-6d1e-1604604a7218/data/hbase/meta/1588230740/ns/af19fb446bae466b91e6dc0aa1a11ef6, entries=2, sequenceid=6, filesize=5.0 K
2024-11-24T04:29:35,054 INFO  [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 51ms, sequenceid=6, compaction requested=false
2024-11-24T04:29:35,054 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-24T04:29:35,060 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40093/user/jenkins/test-data/4b046d87-3de5-4fbe-6d1e-1604604a7218/data/hbase/meta/1588230740/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1
2024-11-24T04:29:35,061 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-24T04:29:35,061 INFO  [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-24T04:29:35,061 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732422575002Running coprocessor pre-close hooks at 1732422575002Disabling compacts and flushes for region at 1732422575002Disabling writes for close at 1732422575003 (+1 ms)Obtaining lock to block concurrent updates at 1732422575003Preparing flush snapshotting stores in 1588230740 at 1732422575003Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=74, getHeapSize=1184, getOffHeapSize=0, getCellsCount=2 at 1732422575003Flushing stores of hbase:meta,,1.1588230740 at 1732422575004 (+1 ms)Flushing 1588230740/ns: creating writer at 1732422575004Flushing 1588230740/ns: appending metadata at 1732422575028 (+24 ms)Flushing 1588230740/ns: closing flushed file at 1732422575028Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@16d487ed: reopening flushed file at 1732422575043 (+15 ms)Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 51ms, sequenceid=6, compaction requested=false at 1732422575054 (+11 ms)Writing region close event to WAL at 1732422575056 (+2 ms)Running coprocessor post-close hooks at 1732422575061 (+5 ms)Closed at 1732422575061
2024-11-24T04:29:35,062 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-24T04:29:35,202 INFO  [RS:0;fadb2fe0ba32:38993 {}] regionserver.HRegionServer(976): stopping server fadb2fe0ba32,38993,1732422573858; all regions closed.
2024-11-24T04:29:35,203 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:35,203 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:35,203 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:35,203 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:35,204 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:35,206 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45271 is added to blk_1073741834_1010 (size=1152)
2024-11-24T04:29:35,206 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45131 is added to blk_1073741834_1010 (size=1152)
2024-11-24T04:29:35,209 DEBUG [RS:0;fadb2fe0ba32:38993 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4b046d87-3de5-4fbe-6d1e-1604604a7218/oldWALs
2024-11-24T04:29:35,209 INFO  [RS:0;fadb2fe0ba32:38993 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog fadb2fe0ba32%2C38993%2C1732422573858.meta:.meta(num 1732422574858)
2024-11-24T04:29:35,209 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:35,209 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:35,210 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:35,210 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:35,210 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:35,212 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45271 is added to blk_1073741833_1009 (size=93)
2024-11-24T04:29:35,213 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45131 is added to blk_1073741833_1009 (size=93)
2024-11-24T04:29:35,214 DEBUG [RS:0;fadb2fe0ba32:38993 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4b046d87-3de5-4fbe-6d1e-1604604a7218/oldWALs
2024-11-24T04:29:35,215 INFO  [RS:0;fadb2fe0ba32:38993 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog fadb2fe0ba32%2C38993%2C1732422573858:(num 1732422574479)
2024-11-24T04:29:35,215 DEBUG [RS:0;fadb2fe0ba32:38993 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T04:29:35,215 INFO  [RS:0;fadb2fe0ba32:38993 {}] regionserver.LeaseManager(133): Closed leases
2024-11-24T04:29:35,215 INFO  [RS:0;fadb2fe0ba32:38993 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-24T04:29:35,215 INFO  [RS:0;fadb2fe0ba32:38993 {}] hbase.ChoreService(370): Chore service for: regionserver/fadb2fe0ba32:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-24T04:29:35,215 INFO  [RS:0;fadb2fe0ba32:38993 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-24T04:29:35,215 INFO  [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-24T04:29:35,215 INFO  [RS:0;fadb2fe0ba32:38993 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38993
2024-11-24T04:29:35,217 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x101483c87bd0001, quorum=127.0.0.1:49303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/fadb2fe0ba32,38993,1732422573858
2024-11-24T04:29:35,217 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39971-0x101483c87bd0000, quorum=127.0.0.1:49303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-24T04:29:35,217 INFO  [RS:0;fadb2fe0ba32:38993 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-24T04:29:35,219 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [fadb2fe0ba32,38993,1732422573858]
2024-11-24T04:29:35,221 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/fadb2fe0ba32,38993,1732422573858 already deleted, retry=false
2024-11-24T04:29:35,222 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; fadb2fe0ba32,38993,1732422573858 expired; onlineServers=0
2024-11-24T04:29:35,222 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'fadb2fe0ba32,39971,1732422573785' *****
2024-11-24T04:29:35,222 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-24T04:29:35,222 INFO  [M:0;fadb2fe0ba32:39971 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-24T04:29:35,222 INFO  [M:0;fadb2fe0ba32:39971 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-24T04:29:35,222 DEBUG [M:0;fadb2fe0ba32:39971 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-24T04:29:35,222 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-24T04:29:35,222 DEBUG [M:0;fadb2fe0ba32:39971 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-24T04:29:35,222 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster-HFileCleaner.small.0-1732422574231 {}] cleaner.HFileCleaner(306): Exit Thread[master/fadb2fe0ba32:0:becomeActiveMaster-HFileCleaner.small.0-1732422574231,5,FailOnTimeoutGroup]
2024-11-24T04:29:35,222 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster-HFileCleaner.large.0-1732422574231 {}] cleaner.HFileCleaner(306): Exit Thread[master/fadb2fe0ba32:0:becomeActiveMaster-HFileCleaner.large.0-1732422574231,5,FailOnTimeoutGroup]
2024-11-24T04:29:35,222 INFO  [M:0;fadb2fe0ba32:39971 {}] hbase.ChoreService(370): Chore service for: master/fadb2fe0ba32:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-24T04:29:35,222 INFO  [M:0;fadb2fe0ba32:39971 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-24T04:29:35,223 DEBUG [M:0;fadb2fe0ba32:39971 {}] master.HMaster(1795): Stopping service threads
2024-11-24T04:29:35,223 INFO  [M:0;fadb2fe0ba32:39971 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-24T04:29:35,223 INFO  [M:0;fadb2fe0ba32:39971 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-24T04:29:35,223 INFO  [M:0;fadb2fe0ba32:39971 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-24T04:29:35,223 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-24T04:29:35,224 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39971-0x101483c87bd0000, quorum=127.0.0.1:49303, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-24T04:29:35,224 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39971-0x101483c87bd0000, quorum=127.0.0.1:49303, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:29:35,224 DEBUG [M:0;fadb2fe0ba32:39971 {}] zookeeper.ZKUtil(347): master:39971-0x101483c87bd0000, quorum=127.0.0.1:49303, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-24T04:29:35,224 WARN  [M:0;fadb2fe0ba32:39971 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-24T04:29:35,225 INFO  [M:0;fadb2fe0ba32:39971 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:40093/user/jenkins/test-data/4b046d87-3de5-4fbe-6d1e-1604604a7218/.lastflushedseqids
2024-11-24T04:29:35,231 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45271 is added to blk_1073741836_1012 (size=99)
2024-11-24T04:29:35,232 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45131 is added to blk_1073741836_1012 (size=99)
2024-11-24T04:29:35,232 INFO  [M:0;fadb2fe0ba32:39971 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-24T04:29:35,232 INFO  [M:0;fadb2fe0ba32:39971 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-24T04:29:35,232 DEBUG [M:0;fadb2fe0ba32:39971 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-24T04:29:35,232 INFO  [M:0;fadb2fe0ba32:39971 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T04:29:35,232 DEBUG [M:0;fadb2fe0ba32:39971 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T04:29:35,232 DEBUG [M:0;fadb2fe0ba32:39971 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-24T04:29:35,232 DEBUG [M:0;fadb2fe0ba32:39971 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T04:29:35,232 INFO  [M:0;fadb2fe0ba32:39971 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.67 KB heapSize=11.34 KB
2024-11-24T04:29:35,251 DEBUG [M:0;fadb2fe0ba32:39971 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40093/user/jenkins/test-data/4b046d87-3de5-4fbe-6d1e-1604604a7218/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/88c7d5c395714561a6ef125d813e46ec is 82, key is hbase:meta,,1/info:regioninfo/1732422574894/Put/seqid=0
2024-11-24T04:29:35,257 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45271 is added to blk_1073741837_1013 (size=5672)
2024-11-24T04:29:35,257 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45131 is added to blk_1073741837_1013 (size=5672)
2024-11-24T04:29:35,258 INFO  [M:0;fadb2fe0ba32:39971 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:40093/user/jenkins/test-data/4b046d87-3de5-4fbe-6d1e-1604604a7218/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/88c7d5c395714561a6ef125d813e46ec
2024-11-24T04:29:35,281 DEBUG [M:0;fadb2fe0ba32:39971 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40093/user/jenkins/test-data/4b046d87-3de5-4fbe-6d1e-1604604a7218/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f0b95b58c2fb4dc7ad28f9ab45742593 is 240, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1732422574921/Put/seqid=0
2024-11-24T04:29:35,286 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45271 is added to blk_1073741838_1014 (size=5275)
2024-11-24T04:29:35,287 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45131 is added to blk_1073741838_1014 (size=5275)
2024-11-24T04:29:35,287 INFO  [M:0;fadb2fe0ba32:39971 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:40093/user/jenkins/test-data/4b046d87-3de5-4fbe-6d1e-1604604a7218/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f0b95b58c2fb4dc7ad28f9ab45742593
2024-11-24T04:29:35,310 DEBUG [M:0;fadb2fe0ba32:39971 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40093/user/jenkins/test-data/4b046d87-3de5-4fbe-6d1e-1604604a7218/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/6d4f9f85d215430f9a6f6b9d43a80655 is 69, key is fadb2fe0ba32,38993,1732422573858/rs:state/1732422574306/Put/seqid=0
2024-11-24T04:29:35,315 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45131 is added to blk_1073741839_1015 (size=5156)
2024-11-24T04:29:35,315 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45271 is added to blk_1073741839_1015 (size=5156)
2024-11-24T04:29:35,315 INFO  [M:0;fadb2fe0ba32:39971 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:40093/user/jenkins/test-data/4b046d87-3de5-4fbe-6d1e-1604604a7218/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/6d4f9f85d215430f9a6f6b9d43a80655
2024-11-24T04:29:35,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x101483c87bd0001, quorum=127.0.0.1:49303, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-24T04:29:35,319 INFO  [RS:0;fadb2fe0ba32:38993 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-24T04:29:35,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38993-0x101483c87bd0001, quorum=127.0.0.1:49303, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-24T04:29:35,319 INFO  [RS:0;fadb2fe0ba32:38993 {}] regionserver.HRegionServer(1031): Exiting; stopping=fadb2fe0ba32,38993,1732422573858; zookeeper connection closed.
2024-11-24T04:29:35,319 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2bfc94dc {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2bfc94dc
2024-11-24T04:29:35,320 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-24T04:29:35,338 DEBUG [M:0;fadb2fe0ba32:39971 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40093/user/jenkins/test-data/4b046d87-3de5-4fbe-6d1e-1604604a7218/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/1b72431eb9f147d299032b409e7dad40 is 52, key is load_balancer_on/state:d/1732422574995/Put/seqid=0
2024-11-24T04:29:35,344 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45271 is added to blk_1073741840_1016 (size=5056)
2024-11-24T04:29:35,344 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45131 is added to blk_1073741840_1016 (size=5056)
2024-11-24T04:29:35,344 INFO  [M:0;fadb2fe0ba32:39971 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:40093/user/jenkins/test-data/4b046d87-3de5-4fbe-6d1e-1604604a7218/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/1b72431eb9f147d299032b409e7dad40
2024-11-24T04:29:35,351 DEBUG [M:0;fadb2fe0ba32:39971 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40093/user/jenkins/test-data/4b046d87-3de5-4fbe-6d1e-1604604a7218/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/88c7d5c395714561a6ef125d813e46ec as hdfs://localhost:40093/user/jenkins/test-data/4b046d87-3de5-4fbe-6d1e-1604604a7218/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/88c7d5c395714561a6ef125d813e46ec
2024-11-24T04:29:35,357 INFO  [M:0;fadb2fe0ba32:39971 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40093/user/jenkins/test-data/4b046d87-3de5-4fbe-6d1e-1604604a7218/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/88c7d5c395714561a6ef125d813e46ec, entries=8, sequenceid=29, filesize=5.5 K
2024-11-24T04:29:35,358 DEBUG [M:0;fadb2fe0ba32:39971 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40093/user/jenkins/test-data/4b046d87-3de5-4fbe-6d1e-1604604a7218/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f0b95b58c2fb4dc7ad28f9ab45742593 as hdfs://localhost:40093/user/jenkins/test-data/4b046d87-3de5-4fbe-6d1e-1604604a7218/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f0b95b58c2fb4dc7ad28f9ab45742593
2024-11-24T04:29:35,363 INFO  [M:0;fadb2fe0ba32:39971 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40093/user/jenkins/test-data/4b046d87-3de5-4fbe-6d1e-1604604a7218/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f0b95b58c2fb4dc7ad28f9ab45742593, entries=3, sequenceid=29, filesize=5.2 K
2024-11-24T04:29:35,365 DEBUG [M:0;fadb2fe0ba32:39971 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40093/user/jenkins/test-data/4b046d87-3de5-4fbe-6d1e-1604604a7218/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/6d4f9f85d215430f9a6f6b9d43a80655 as hdfs://localhost:40093/user/jenkins/test-data/4b046d87-3de5-4fbe-6d1e-1604604a7218/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/6d4f9f85d215430f9a6f6b9d43a80655
2024-11-24T04:29:35,370 INFO  [M:0;fadb2fe0ba32:39971 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40093/user/jenkins/test-data/4b046d87-3de5-4fbe-6d1e-1604604a7218/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/6d4f9f85d215430f9a6f6b9d43a80655, entries=1, sequenceid=29, filesize=5.0 K
2024-11-24T04:29:35,371 DEBUG [M:0;fadb2fe0ba32:39971 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40093/user/jenkins/test-data/4b046d87-3de5-4fbe-6d1e-1604604a7218/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/1b72431eb9f147d299032b409e7dad40 as hdfs://localhost:40093/user/jenkins/test-data/4b046d87-3de5-4fbe-6d1e-1604604a7218/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/1b72431eb9f147d299032b409e7dad40
2024-11-24T04:29:35,376 INFO  [M:0;fadb2fe0ba32:39971 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40093/user/jenkins/test-data/4b046d87-3de5-4fbe-6d1e-1604604a7218/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/1b72431eb9f147d299032b409e7dad40, entries=1, sequenceid=29, filesize=4.9 K
2024-11-24T04:29:35,378 INFO  [M:0;fadb2fe0ba32:39971 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 146ms, sequenceid=29, compaction requested=false
2024-11-24T04:29:35,379 INFO  [M:0;fadb2fe0ba32:39971 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T04:29:35,380 DEBUG [M:0;fadb2fe0ba32:39971 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732422575232Disabling compacts and flushes for region at 1732422575232Disabling writes for close at 1732422575232Obtaining lock to block concurrent updates at 1732422575232Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732422575232Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7850, getHeapSize=11544, getOffHeapSize=0, getCellsCount=36 at 1732422575233 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732422575234 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732422575234Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732422575250 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732422575250Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732422575264 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732422575280 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732422575281 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732422575293 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732422575309 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732422575309Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732422575321 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732422575338 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732422575338Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1ca14c5: reopening flushed file at 1732422575350 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6629a41a: reopening flushed file at 1732422575357 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@581db3bd: reopening flushed file at 1732422575364 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2beaa769: reopening flushed file at 1732422575370 (+6 ms)Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 146ms, sequenceid=29, compaction requested=false at 1732422575378 (+8 ms)Writing region close event to WAL at 1732422575379 (+1 ms)Closed at 1732422575379
2024-11-24T04:29:35,381 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:35,381 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:35,381 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:35,382 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:35,382 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:35,384 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45131 is added to blk_1073741830_1006 (size=10311)
2024-11-24T04:29:35,384 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45271 is added to blk_1073741830_1006 (size=10311)
2024-11-24T04:29:35,385 INFO  [M:0;fadb2fe0ba32:39971 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-24T04:29:35,385 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-24T04:29:35,385 INFO  [M:0;fadb2fe0ba32:39971 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39971
2024-11-24T04:29:35,386 INFO  [M:0;fadb2fe0ba32:39971 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-24T04:29:35,488 INFO  [M:0;fadb2fe0ba32:39971 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-24T04:29:35,488 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39971-0x101483c87bd0000, quorum=127.0.0.1:49303, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-24T04:29:35,489 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39971-0x101483c87bd0000, quorum=127.0.0.1:49303, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-24T04:29:35,491 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@824b6ae{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T04:29:35,492 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5d7e5c06{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-24T04:29:35,492 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-24T04:29:35,492 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@376d199b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-24T04:29:35,492 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2b44e274{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85930ece-2ad7-a1e7-d45a-6b50e739afc7/hadoop.log.dir/,STOPPED}
2024-11-24T04:29:35,494 WARN  [BP-1338711505-172.17.0.2-1732422572357 heartbeating to localhost/127.0.0.1:40093 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-24T04:29:35,494 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-24T04:29:35,494 WARN  [BP-1338711505-172.17.0.2-1732422572357 heartbeating to localhost/127.0.0.1:40093 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1338711505-172.17.0.2-1732422572357 (Datanode Uuid de4c9845-b483-4524-ad1a-5131a6b4bde0) service to localhost/127.0.0.1:40093
2024-11-24T04:29:35,494 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-24T04:29:35,494 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85930ece-2ad7-a1e7-d45a-6b50e739afc7/cluster_bd22959d-b714-a1e2-e9d7-576ceba76b35/data/data3/current/BP-1338711505-172.17.0.2-1732422572357 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T04:29:35,494 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85930ece-2ad7-a1e7-d45a-6b50e739afc7/cluster_bd22959d-b714-a1e2-e9d7-576ceba76b35/data/data4/current/BP-1338711505-172.17.0.2-1732422572357 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T04:29:35,495 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-24T04:29:35,497 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5d4bdc00{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T04:29:35,497 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@687b21ce{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-24T04:29:35,497 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-24T04:29:35,497 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@276f8783{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-24T04:29:35,497 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4edee9ab{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85930ece-2ad7-a1e7-d45a-6b50e739afc7/hadoop.log.dir/,STOPPED}
2024-11-24T04:29:35,499 WARN  [BP-1338711505-172.17.0.2-1732422572357 heartbeating to localhost/127.0.0.1:40093 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-24T04:29:35,499 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-24T04:29:35,499 WARN  [BP-1338711505-172.17.0.2-1732422572357 heartbeating to localhost/127.0.0.1:40093 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1338711505-172.17.0.2-1732422572357 (Datanode Uuid b0b5af9f-26ff-4e56-a0b4-117010d9b2b0) service to localhost/127.0.0.1:40093
2024-11-24T04:29:35,499 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-24T04:29:35,499 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85930ece-2ad7-a1e7-d45a-6b50e739afc7/cluster_bd22959d-b714-a1e2-e9d7-576ceba76b35/data/data1/current/BP-1338711505-172.17.0.2-1732422572357 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T04:29:35,500 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85930ece-2ad7-a1e7-d45a-6b50e739afc7/cluster_bd22959d-b714-a1e2-e9d7-576ceba76b35/data/data2/current/BP-1338711505-172.17.0.2-1732422572357 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T04:29:35,500 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-24T04:29:35,506 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@511dc70f{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-24T04:29:35,506 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3e469283{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-24T04:29:35,506 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-24T04:29:35,506 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@45e3157d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-24T04:29:35,507 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3197ca45{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85930ece-2ad7-a1e7-d45a-6b50e739afc7/hadoop.log.dir/,STOPPED}
2024-11-24T04:29:35,515 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-24T04:29:35,536 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-24T04:29:35,536 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-24T04:29:35,536 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85930ece-2ad7-a1e7-d45a-6b50e739afc7/hadoop.log.dir so I do NOT create it in target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1
2024-11-24T04:29:35,536 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/85930ece-2ad7-a1e7-d45a-6b50e739afc7/hadoop.tmp.dir so I do NOT create it in target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1
2024-11-24T04:29:35,536 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/cluster_56272640-4b45-3980-6187-aab4c21513ae, deleteOnExit=true
2024-11-24T04:29:35,536 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-24T04:29:35,536 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/test.cache.data in system properties and HBase conf
2024-11-24T04:29:35,537 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/hadoop.tmp.dir in system properties and HBase conf
2024-11-24T04:29:35,537 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/hadoop.log.dir in system properties and HBase conf
2024-11-24T04:29:35,537 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-24T04:29:35,537 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-24T04:29:35,537 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-24T04:29:35,537 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-24T04:29:35,537 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-24T04:29:35,537 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-24T04:29:35,537 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-24T04:29:35,537 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-24T04:29:35,538 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-24T04:29:35,538 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-24T04:29:35,538 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-24T04:29:35,538 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-24T04:29:35,538 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-24T04:29:35,538 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/nfs.dump.dir in system properties and HBase conf
2024-11-24T04:29:35,538 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/java.io.tmpdir in system properties and HBase conf
2024-11-24T04:29:35,538 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-24T04:29:35,538 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-24T04:29:35,538 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-24T04:29:35,553 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-24T04:29:35,635 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T04:29:35,640 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-24T04:29:35,641 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-24T04:29:35,641 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-24T04:29:35,641 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-24T04:29:35,642 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T04:29:35,642 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6a488aac{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/hadoop.log.dir/,AVAILABLE}
2024-11-24T04:29:35,643 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3ed3a961{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-24T04:29:35,759 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7cd2a640{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/java.io.tmpdir/jetty-localhost-37485-hadoop-hdfs-3_4_1-tests_jar-_-any-4305501250901685575/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-24T04:29:35,759 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@64b7b556{HTTP/1.1, (http/1.1)}{localhost:37485}
2024-11-24T04:29:35,759 INFO  [Time-limited test {}] server.Server(415): Started @104821ms
2024-11-24T04:29:35,774 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-24T04:29:35,846 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T04:29:35,850 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-24T04:29:35,851 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-24T04:29:35,851 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-24T04:29:35,851 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-24T04:29:35,851 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@257cf4bb{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/hadoop.log.dir/,AVAILABLE}
2024-11-24T04:29:35,852 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@30e7c448{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-24T04:29:35,968 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@ab5393f{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/java.io.tmpdir/jetty-localhost-42379-hadoop-hdfs-3_4_1-tests_jar-_-any-14191605690413643706/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T04:29:35,968 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@47f82e76{HTTP/1.1, (http/1.1)}{localhost:42379}
2024-11-24T04:29:35,968 INFO  [Time-limited test {}] server.Server(415): Started @105030ms
2024-11-24T04:29:35,970 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-24T04:29:36,009 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T04:29:36,014 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-24T04:29:36,019 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-24T04:29:36,019 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-24T04:29:36,019 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-24T04:29:36,020 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@69bbaec1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/hadoop.log.dir/,AVAILABLE}
2024-11-24T04:29:36,020 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7720beab{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-24T04:29:36,081 WARN  [Thread-657 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/cluster_56272640-4b45-3980-6187-aab4c21513ae/data/data1/current/BP-1909738581-172.17.0.2-1732422575571/current, will proceed with Du for space computation calculation, 
2024-11-24T04:29:36,081 WARN  [Thread-658 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/cluster_56272640-4b45-3980-6187-aab4c21513ae/data/data2/current/BP-1909738581-172.17.0.2-1732422575571/current, will proceed with Du for space computation calculation, 
2024-11-24T04:29:36,099 WARN  [Thread-636 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-24T04:29:36,103 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe800f0b11d31202d with lease ID 0xbac364592180ee84: Processing first storage report for DS-ebb5afcd-915f-47cc-8761-64b36c291deb from datanode DatanodeRegistration(127.0.0.1:46879, datanodeUuid=2fcf51d3-1c3d-4734-9f68-e9df11c392d4, infoPort=41079, infoSecurePort=0, ipcPort=46669, storageInfo=lv=-57;cid=testClusterID;nsid=612618959;c=1732422575571)
2024-11-24T04:29:36,103 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe800f0b11d31202d with lease ID 0xbac364592180ee84: from storage DS-ebb5afcd-915f-47cc-8761-64b36c291deb node DatanodeRegistration(127.0.0.1:46879, datanodeUuid=2fcf51d3-1c3d-4734-9f68-e9df11c392d4, infoPort=41079, infoSecurePort=0, ipcPort=46669, storageInfo=lv=-57;cid=testClusterID;nsid=612618959;c=1732422575571), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T04:29:36,103 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe800f0b11d31202d with lease ID 0xbac364592180ee84: Processing first storage report for DS-8e5c93b6-3450-42d9-94c8-363433953b0d from datanode DatanodeRegistration(127.0.0.1:46879, datanodeUuid=2fcf51d3-1c3d-4734-9f68-e9df11c392d4, infoPort=41079, infoSecurePort=0, ipcPort=46669, storageInfo=lv=-57;cid=testClusterID;nsid=612618959;c=1732422575571)
2024-11-24T04:29:36,103 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe800f0b11d31202d with lease ID 0xbac364592180ee84: from storage DS-8e5c93b6-3450-42d9-94c8-363433953b0d node DatanodeRegistration(127.0.0.1:46879, datanodeUuid=2fcf51d3-1c3d-4734-9f68-e9df11c392d4, infoPort=41079, infoSecurePort=0, ipcPort=46669, storageInfo=lv=-57;cid=testClusterID;nsid=612618959;c=1732422575571), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T04:29:36,167 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@21d5e4af{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/java.io.tmpdir/jetty-localhost-38185-hadoop-hdfs-3_4_1-tests_jar-_-any-9713178384035441263/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T04:29:36,168 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@145c0180{HTTP/1.1, (http/1.1)}{localhost:38185}
2024-11-24T04:29:36,168 INFO  [Time-limited test {}] server.Server(415): Started @105229ms
2024-11-24T04:29:36,170 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-24T04:29:36,266 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-24T04:29:36,267 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-24T04:29:36,268 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-24T04:29:36,268 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling
2024-11-24T04:29:36,278 WARN  [Thread-684 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/cluster_56272640-4b45-3980-6187-aab4c21513ae/data/data4/current/BP-1909738581-172.17.0.2-1732422575571/current, will proceed with Du for space computation calculation, 
2024-11-24T04:29:36,278 WARN  [Thread-683 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/cluster_56272640-4b45-3980-6187-aab4c21513ae/data/data3/current/BP-1909738581-172.17.0.2-1732422575571/current, will proceed with Du for space computation calculation, 
2024-11-24T04:29:36,303 WARN  [Thread-672 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-24T04:29:36,306 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x219dd6fafa48f0e1 with lease ID 0xbac364592180ee85: Processing first storage report for DS-1c26aeaf-2106-4f04-a38e-568cd2360db7 from datanode DatanodeRegistration(127.0.0.1:37029, datanodeUuid=e3eba06e-e494-45bd-a171-1b16a8edcca7, infoPort=46579, infoSecurePort=0, ipcPort=41557, storageInfo=lv=-57;cid=testClusterID;nsid=612618959;c=1732422575571)
2024-11-24T04:29:36,306 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x219dd6fafa48f0e1 with lease ID 0xbac364592180ee85: from storage DS-1c26aeaf-2106-4f04-a38e-568cd2360db7 node DatanodeRegistration(127.0.0.1:37029, datanodeUuid=e3eba06e-e494-45bd-a171-1b16a8edcca7, infoPort=46579, infoSecurePort=0, ipcPort=41557, storageInfo=lv=-57;cid=testClusterID;nsid=612618959;c=1732422575571), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T04:29:36,306 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x219dd6fafa48f0e1 with lease ID 0xbac364592180ee85: Processing first storage report for DS-0643a443-b5e3-4fc7-8810-801c75a2afdd from datanode DatanodeRegistration(127.0.0.1:37029, datanodeUuid=e3eba06e-e494-45bd-a171-1b16a8edcca7, infoPort=46579, infoSecurePort=0, ipcPort=41557, storageInfo=lv=-57;cid=testClusterID;nsid=612618959;c=1732422575571)
2024-11-24T04:29:36,306 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x219dd6fafa48f0e1 with lease ID 0xbac364592180ee85: from storage DS-0643a443-b5e3-4fc7-8810-801c75a2afdd node DatanodeRegistration(127.0.0.1:37029, datanodeUuid=e3eba06e-e494-45bd-a171-1b16a8edcca7, infoPort=46579, infoSecurePort=0, ipcPort=41557, storageInfo=lv=-57;cid=testClusterID;nsid=612618959;c=1732422575571), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T04:29:36,325 INFO  [regionserver/fadb2fe0ba32:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-24T04:29:36,398 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1
2024-11-24T04:29:36,401 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/cluster_56272640-4b45-3980-6187-aab4c21513ae/zookeeper_0, clientPort=62127, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/cluster_56272640-4b45-3980-6187-aab4c21513ae/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/cluster_56272640-4b45-3980-6187-aab4c21513ae/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-24T04:29:36,402 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=62127
2024-11-24T04:29:36,402 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T04:29:36,404 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T04:29:36,414 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37029 is added to blk_1073741825_1001 (size=7)
2024-11-24T04:29:36,415 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46879 is added to blk_1073741825_1001 (size=7)
2024-11-24T04:29:36,416 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc with version=8
2024-11-24T04:29:36,416 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/hbase-staging
2024-11-24T04:29:36,419 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/fadb2fe0ba32:0 server-side Connection retries=45
2024-11-24T04:29:36,419 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-24T04:29:36,419 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-24T04:29:36,419 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-24T04:29:36,419 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-24T04:29:36,419 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-24T04:29:36,419 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-24T04:29:36,420 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-24T04:29:36,420 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35377
2024-11-24T04:29:36,422 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35377 connecting to ZooKeeper ensemble=127.0.0.1:62127
2024-11-24T04:29:36,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:353770x0, quorum=127.0.0.1:62127, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-24T04:29:36,429 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35377-0x101483c93a00000 connected
2024-11-24T04:29:36,450 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T04:29:36,451 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T04:29:36,454 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35377-0x101483c93a00000, quorum=127.0.0.1:62127, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-24T04:29:36,454 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc, hbase.cluster.distributed=false
2024-11-24T04:29:36,456 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35377-0x101483c93a00000, quorum=127.0.0.1:62127, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-24T04:29:36,460 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35377
2024-11-24T04:29:36,460 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35377
2024-11-24T04:29:36,461 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35377
2024-11-24T04:29:36,464 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35377
2024-11-24T04:29:36,464 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35377
2024-11-24T04:29:36,482 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/fadb2fe0ba32:0 server-side Connection retries=45
2024-11-24T04:29:36,482 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-24T04:29:36,482 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-24T04:29:36,482 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-24T04:29:36,482 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-24T04:29:36,482 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-24T04:29:36,482 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-24T04:29:36,482 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-24T04:29:36,483 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40681
2024-11-24T04:29:36,485 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40681 connecting to ZooKeeper ensemble=127.0.0.1:62127
2024-11-24T04:29:36,485 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T04:29:36,487 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T04:29:36,492 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:406810x0, quorum=127.0.0.1:62127, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-24T04:29:36,493 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:406810x0, quorum=127.0.0.1:62127, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-24T04:29:36,493 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40681-0x101483c93a00001 connected
2024-11-24T04:29:36,493 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-24T04:29:36,494 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-24T04:29:36,494 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40681-0x101483c93a00001, quorum=127.0.0.1:62127, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-24T04:29:36,495 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40681-0x101483c93a00001, quorum=127.0.0.1:62127, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-24T04:29:36,497 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40681
2024-11-24T04:29:36,497 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40681
2024-11-24T04:29:36,497 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40681
2024-11-24T04:29:36,498 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40681
2024-11-24T04:29:36,498 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40681
2024-11-24T04:29:36,511 DEBUG [M:0;fadb2fe0ba32:35377 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;fadb2fe0ba32:35377
2024-11-24T04:29:36,512 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/fadb2fe0ba32,35377,1732422576418
2024-11-24T04:29:36,513 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35377-0x101483c93a00000, quorum=127.0.0.1:62127, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-24T04:29:36,513 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40681-0x101483c93a00001, quorum=127.0.0.1:62127, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-24T04:29:36,514 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35377-0x101483c93a00000, quorum=127.0.0.1:62127, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/fadb2fe0ba32,35377,1732422576418
2024-11-24T04:29:36,515 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35377-0x101483c93a00000, quorum=127.0.0.1:62127, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:29:36,516 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40681-0x101483c93a00001, quorum=127.0.0.1:62127, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-24T04:29:36,516 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40681-0x101483c93a00001, quorum=127.0.0.1:62127, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:29:36,517 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35377-0x101483c93a00000, quorum=127.0.0.1:62127, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-24T04:29:36,518 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/fadb2fe0ba32,35377,1732422576418 from backup master directory
2024-11-24T04:29:36,520 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35377-0x101483c93a00000, quorum=127.0.0.1:62127, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/fadb2fe0ba32,35377,1732422576418
2024-11-24T04:29:36,520 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40681-0x101483c93a00001, quorum=127.0.0.1:62127, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-24T04:29:36,520 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35377-0x101483c93a00000, quorum=127.0.0.1:62127, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-24T04:29:36,521 WARN  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-24T04:29:36,521 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=fadb2fe0ba32,35377,1732422576418
2024-11-24T04:29:36,522 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:29:36,527 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/hbase.id] with ID: 4e7a7841-646c-4aea-b78c-b3202165c79a
2024-11-24T04:29:36,527 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/.tmp/hbase.id
2024-11-24T04:29:36,528 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:29:36,543 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46879 is added to blk_1073741826_1002 (size=42)
2024-11-24T04:29:36,544 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37029 is added to blk_1073741826_1002 (size=42)
2024-11-24T04:29:36,544 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/.tmp/hbase.id]:[hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/hbase.id]
2024-11-24T04:29:36,562 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T04:29:36,563 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-24T04:29:36,564 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-11-24T04:29:36,566 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40681-0x101483c93a00001, quorum=127.0.0.1:62127, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:29:36,566 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35377-0x101483c93a00000, quorum=127.0.0.1:62127, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:29:36,583 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46879 is added to blk_1073741827_1003 (size=196)
2024-11-24T04:29:36,583 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37029 is added to blk_1073741827_1003 (size=196)
2024-11-24T04:29:36,584 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-24T04:29:36,585 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-24T04:29:36,586 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-24T04:29:36,598 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37029 is added to blk_1073741828_1004 (size=1189)
2024-11-24T04:29:36,598 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46879 is added to blk_1073741828_1004 (size=1189)
2024-11-24T04:29:36,600 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/MasterData/data/master/store
2024-11-24T04:29:36,608 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46879 is added to blk_1073741829_1005 (size=34)
2024-11-24T04:29:36,609 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37029 is added to blk_1073741829_1005 (size=34)
2024-11-24T04:29:36,609 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T04:29:36,609 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-24T04:29:36,610 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T04:29:36,610 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T04:29:36,610 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-24T04:29:36,610 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T04:29:36,610 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T04:29:36,610 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732422576609Disabling compacts and flushes for region at 1732422576609Disabling writes for close at 1732422576610 (+1 ms)Writing region close event to WAL at 1732422576610Closed at 1732422576610
2024-11-24T04:29:36,611 WARN  [master/fadb2fe0ba32:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/MasterData/data/master/store/.initializing
2024-11-24T04:29:36,611 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/MasterData/WALs/fadb2fe0ba32,35377,1732422576418
2024-11-24T04:29:36,614 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fadb2fe0ba32%2C35377%2C1732422576418, suffix=, logDir=hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/MasterData/WALs/fadb2fe0ba32,35377,1732422576418, archiveDir=hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/MasterData/oldWALs, maxLogs=10
2024-11-24T04:29:36,614 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor fadb2fe0ba32%2C35377%2C1732422576418.1732422576614
2024-11-24T04:29:36,624 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/MasterData/WALs/fadb2fe0ba32,35377,1732422576418/fadb2fe0ba32%2C35377%2C1732422576418.1732422576614
2024-11-24T04:29:36,631 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46579:46579),(127.0.0.1/127.0.0.1:41079:41079)]
2024-11-24T04:29:36,631 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-24T04:29:36,632 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T04:29:36,632 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T04:29:36,632 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T04:29:36,634 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-24T04:29:36,636 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-24T04:29:36,636 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:29:36,636 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T04:29:36,636 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-24T04:29:36,638 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-24T04:29:36,638 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:29:36,639 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-24T04:29:36,639 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-24T04:29:36,640 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-24T04:29:36,640 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:29:36,641 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-24T04:29:36,641 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-24T04:29:36,642 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-24T04:29:36,642 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:29:36,643 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-24T04:29:36,643 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T04:29:36,644 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-24T04:29:36,644 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-24T04:29:36,645 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T04:29:36,646 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T04:29:36,646 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-24T04:29:36,647 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T04:29:36,650 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-24T04:29:36,650 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=770195, jitterRate=-0.020647525787353516}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-24T04:29:36,651 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1732422576632Initializing all the Stores at 1732422576633 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732422576633Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422576634 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422576634Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422576634Cleaning up temporary data from old regions at 1732422576646 (+12 ms)Region opened successfully at 1732422576651 (+5 ms)
2024-11-24T04:29:36,655 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-24T04:29:36,659 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f991603, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fadb2fe0ba32/172.17.0.2:0
2024-11-24T04:29:36,660 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-24T04:29:36,660 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-24T04:29:36,660 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-24T04:29:36,660 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-24T04:29:36,660 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-24T04:29:36,661 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-24T04:29:36,661 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-24T04:29:36,670 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-24T04:29:36,671 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35377-0x101483c93a00000, quorum=127.0.0.1:62127, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-24T04:29:36,682 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-24T04:29:36,682 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-24T04:29:36,683 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35377-0x101483c93a00000, quorum=127.0.0.1:62127, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-24T04:29:36,692 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-24T04:29:36,692 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-24T04:29:36,694 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35377-0x101483c93a00000, quorum=127.0.0.1:62127, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-24T04:29:36,696 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-24T04:29:36,698 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35377-0x101483c93a00000, quorum=127.0.0.1:62127, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-24T04:29:36,699 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-24T04:29:36,702 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35377-0x101483c93a00000, quorum=127.0.0.1:62127, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-24T04:29:36,703 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-24T04:29:36,705 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40681-0x101483c93a00001, quorum=127.0.0.1:62127, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-24T04:29:36,705 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35377-0x101483c93a00000, quorum=127.0.0.1:62127, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-24T04:29:36,705 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40681-0x101483c93a00001, quorum=127.0.0.1:62127, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:29:36,705 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35377-0x101483c93a00000, quorum=127.0.0.1:62127, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:29:36,706 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=fadb2fe0ba32,35377,1732422576418, sessionid=0x101483c93a00000, setting cluster-up flag (Was=false)
2024-11-24T04:29:36,709 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35377-0x101483c93a00000, quorum=127.0.0.1:62127, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:29:36,710 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40681-0x101483c93a00001, quorum=127.0.0.1:62127, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:29:36,717 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-24T04:29:36,718 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=fadb2fe0ba32,35377,1732422576418
2024-11-24T04:29:36,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35377-0x101483c93a00000, quorum=127.0.0.1:62127, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:29:36,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40681-0x101483c93a00001, quorum=127.0.0.1:62127, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:29:36,730 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-24T04:29:36,731 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=fadb2fe0ba32,35377,1732422576418
2024-11-24T04:29:36,733 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-24T04:29:36,735 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-24T04:29:36,735 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-24T04:29:36,735 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-24T04:29:36,735 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: fadb2fe0ba32,35377,1732422576418
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-24T04:29:36,737 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/fadb2fe0ba32:0, corePoolSize=5, maxPoolSize=5
2024-11-24T04:29:36,737 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/fadb2fe0ba32:0, corePoolSize=5, maxPoolSize=5
2024-11-24T04:29:36,737 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/fadb2fe0ba32:0, corePoolSize=5, maxPoolSize=5
2024-11-24T04:29:36,737 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/fadb2fe0ba32:0, corePoolSize=5, maxPoolSize=5
2024-11-24T04:29:36,737 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/fadb2fe0ba32:0, corePoolSize=10, maxPoolSize=10
2024-11-24T04:29:36,737 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:29:36,737 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/fadb2fe0ba32:0, corePoolSize=2, maxPoolSize=2
2024-11-24T04:29:36,737 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:29:36,739 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-24T04:29:36,739 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-24T04:29:36,740 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:29:36,740 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-24T04:29:36,749 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732422606748
2024-11-24T04:29:36,749 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-24T04:29:36,749 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-24T04:29:36,749 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-24T04:29:36,749 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-24T04:29:36,749 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-24T04:29:36,749 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-24T04:29:36,749 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-24T04:29:36,750 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-24T04:29:36,750 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-24T04:29:36,751 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-24T04:29:36,755 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-24T04:29:36,755 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-24T04:29:36,755 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/fadb2fe0ba32:0:becomeActiveMaster-HFileCleaner.large.0-1732422576755,5,FailOnTimeoutGroup]
2024-11-24T04:29:36,755 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/fadb2fe0ba32:0:becomeActiveMaster-HFileCleaner.small.0-1732422576755,5,FailOnTimeoutGroup]
2024-11-24T04:29:36,756 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-24T04:29:36,756 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-24T04:29:36,756 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-24T04:29:36,756 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-24T04:29:36,758 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37029 is added to blk_1073741831_1007 (size=1321)
2024-11-24T04:29:36,758 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46879 is added to blk_1073741831_1007 (size=1321)
2024-11-24T04:29:36,759 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-24T04:29:36,759 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc
2024-11-24T04:29:36,766 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37029 is added to blk_1073741832_1008 (size=32)
2024-11-24T04:29:36,766 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46879 is added to blk_1073741832_1008 (size=32)
2024-11-24T04:29:36,767 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T04:29:36,768 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-24T04:29:36,770 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-24T04:29:36,770 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:29:36,770 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T04:29:36,771 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-24T04:29:36,772 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-24T04:29:36,772 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:29:36,772 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T04:29:36,773 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-24T04:29:36,774 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-24T04:29:36,774 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:29:36,775 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T04:29:36,775 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-24T04:29:36,776 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-24T04:29:36,776 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:29:36,776 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T04:29:36,777 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-24T04:29:36,777 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/hbase/meta/1588230740
2024-11-24T04:29:36,778 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/hbase/meta/1588230740
2024-11-24T04:29:36,779 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-24T04:29:36,779 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-24T04:29:36,780 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-24T04:29:36,781 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-24T04:29:36,783 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-24T04:29:36,784 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=864366, jitterRate=0.09909935295581818}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-24T04:29:36,785 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1732422576767Initializing all the Stores at 1732422576768 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732422576768Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732422576768Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422576768Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732422576768Cleaning up temporary data from old regions at 1732422576779 (+11 ms)Region opened successfully at 1732422576785 (+6 ms)
2024-11-24T04:29:36,785 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-24T04:29:36,785 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-24T04:29:36,785 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-24T04:29:36,785 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-24T04:29:36,785 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-24T04:29:36,785 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-24T04:29:36,786 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732422576785Disabling compacts and flushes for region at 1732422576785Disabling writes for close at 1732422576785Writing region close event to WAL at 1732422576785Closed at 1732422576785
2024-11-24T04:29:36,787 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-24T04:29:36,787 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-24T04:29:36,787 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-24T04:29:36,789 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-24T04:29:36,790 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-24T04:29:36,800 INFO  [RS:0;fadb2fe0ba32:40681 {}] regionserver.HRegionServer(746): ClusterId : 4e7a7841-646c-4aea-b78c-b3202165c79a
2024-11-24T04:29:36,800 DEBUG [RS:0;fadb2fe0ba32:40681 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-24T04:29:36,802 DEBUG [RS:0;fadb2fe0ba32:40681 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-24T04:29:36,802 DEBUG [RS:0;fadb2fe0ba32:40681 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-24T04:29:36,804 DEBUG [RS:0;fadb2fe0ba32:40681 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-24T04:29:36,804 DEBUG [RS:0;fadb2fe0ba32:40681 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@aa1ed50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fadb2fe0ba32/172.17.0.2:0
2024-11-24T04:29:36,818 DEBUG [RS:0;fadb2fe0ba32:40681 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;fadb2fe0ba32:40681
2024-11-24T04:29:36,818 INFO  [RS:0;fadb2fe0ba32:40681 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-24T04:29:36,818 INFO  [RS:0;fadb2fe0ba32:40681 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-24T04:29:36,818 DEBUG [RS:0;fadb2fe0ba32:40681 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-24T04:29:36,819 INFO  [RS:0;fadb2fe0ba32:40681 {}] regionserver.HRegionServer(2659): reportForDuty to master=fadb2fe0ba32,35377,1732422576418 with port=40681, startcode=1732422576481
2024-11-24T04:29:36,819 DEBUG [RS:0;fadb2fe0ba32:40681 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-24T04:29:36,822 INFO  [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51797, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService
2024-11-24T04:29:36,822 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35377 {}] master.ServerManager(363): Checking decommissioned status of RegionServer fadb2fe0ba32,40681,1732422576481
2024-11-24T04:29:36,822 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35377 {}] master.ServerManager(517): Registering regionserver=fadb2fe0ba32,40681,1732422576481
2024-11-24T04:29:36,827 DEBUG [RS:0;fadb2fe0ba32:40681 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc
2024-11-24T04:29:36,827 DEBUG [RS:0;fadb2fe0ba32:40681 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40187
2024-11-24T04:29:36,828 DEBUG [RS:0;fadb2fe0ba32:40681 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-24T04:29:36,831 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35377-0x101483c93a00000, quorum=127.0.0.1:62127, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-24T04:29:36,832 DEBUG [RS:0;fadb2fe0ba32:40681 {}] zookeeper.ZKUtil(111): regionserver:40681-0x101483c93a00001, quorum=127.0.0.1:62127, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/fadb2fe0ba32,40681,1732422576481
2024-11-24T04:29:36,832 WARN  [RS:0;fadb2fe0ba32:40681 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-24T04:29:36,833 INFO  [RS:0;fadb2fe0ba32:40681 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-24T04:29:36,833 DEBUG [RS:0;fadb2fe0ba32:40681 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481
2024-11-24T04:29:36,835 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [fadb2fe0ba32,40681,1732422576481]
2024-11-24T04:29:36,838 INFO  [RS:0;fadb2fe0ba32:40681 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-24T04:29:36,840 INFO  [RS:0;fadb2fe0ba32:40681 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-24T04:29:36,841 INFO  [RS:0;fadb2fe0ba32:40681 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-24T04:29:36,841 INFO  [RS:0;fadb2fe0ba32:40681 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-24T04:29:36,850 INFO  [RS:0;fadb2fe0ba32:40681 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-24T04:29:36,851 INFO  [RS:0;fadb2fe0ba32:40681 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-24T04:29:36,851 INFO  [RS:0;fadb2fe0ba32:40681 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-24T04:29:36,852 DEBUG [RS:0;fadb2fe0ba32:40681 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:29:36,852 DEBUG [RS:0;fadb2fe0ba32:40681 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:29:36,852 DEBUG [RS:0;fadb2fe0ba32:40681 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:29:36,852 DEBUG [RS:0;fadb2fe0ba32:40681 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:29:36,852 DEBUG [RS:0;fadb2fe0ba32:40681 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:29:36,852 DEBUG [RS:0;fadb2fe0ba32:40681 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/fadb2fe0ba32:0, corePoolSize=2, maxPoolSize=2
2024-11-24T04:29:36,852 DEBUG [RS:0;fadb2fe0ba32:40681 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:29:36,852 DEBUG [RS:0;fadb2fe0ba32:40681 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:29:36,852 DEBUG [RS:0;fadb2fe0ba32:40681 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:29:36,853 DEBUG [RS:0;fadb2fe0ba32:40681 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:29:36,853 DEBUG [RS:0;fadb2fe0ba32:40681 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:29:36,853 DEBUG [RS:0;fadb2fe0ba32:40681 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:29:36,853 DEBUG [RS:0;fadb2fe0ba32:40681 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/fadb2fe0ba32:0, corePoolSize=3, maxPoolSize=3
2024-11-24T04:29:36,853 DEBUG [RS:0;fadb2fe0ba32:40681 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/fadb2fe0ba32:0, corePoolSize=3, maxPoolSize=3
2024-11-24T04:29:36,853 INFO  [RS:0;fadb2fe0ba32:40681 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-24T04:29:36,854 INFO  [RS:0;fadb2fe0ba32:40681 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-24T04:29:36,854 INFO  [RS:0;fadb2fe0ba32:40681 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-24T04:29:36,854 INFO  [RS:0;fadb2fe0ba32:40681 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-24T04:29:36,854 INFO  [RS:0;fadb2fe0ba32:40681 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-24T04:29:36,854 INFO  [RS:0;fadb2fe0ba32:40681 {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,40681,1732422576481-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-24T04:29:36,871 INFO  [RS:0;fadb2fe0ba32:40681 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-24T04:29:36,871 INFO  [RS:0;fadb2fe0ba32:40681 {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,40681,1732422576481-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-24T04:29:36,871 INFO  [RS:0;fadb2fe0ba32:40681 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T04:29:36,871 INFO  [RS:0;fadb2fe0ba32:40681 {}] regionserver.Replication(171): fadb2fe0ba32,40681,1732422576481 started
2024-11-24T04:29:36,888 INFO  [RS:0;fadb2fe0ba32:40681 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T04:29:36,888 INFO  [RS:0;fadb2fe0ba32:40681 {}] regionserver.HRegionServer(1482): Serving as fadb2fe0ba32,40681,1732422576481, RpcServer on fadb2fe0ba32/172.17.0.2:40681, sessionid=0x101483c93a00001
2024-11-24T04:29:36,888 DEBUG [RS:0;fadb2fe0ba32:40681 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-24T04:29:36,888 DEBUG [RS:0;fadb2fe0ba32:40681 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager fadb2fe0ba32,40681,1732422576481
2024-11-24T04:29:36,888 DEBUG [RS:0;fadb2fe0ba32:40681 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fadb2fe0ba32,40681,1732422576481'
2024-11-24T04:29:36,889 DEBUG [RS:0;fadb2fe0ba32:40681 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-24T04:29:36,889 DEBUG [RS:0;fadb2fe0ba32:40681 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-24T04:29:36,890 DEBUG [RS:0;fadb2fe0ba32:40681 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-24T04:29:36,890 DEBUG [RS:0;fadb2fe0ba32:40681 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-24T04:29:36,890 DEBUG [RS:0;fadb2fe0ba32:40681 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager fadb2fe0ba32,40681,1732422576481
2024-11-24T04:29:36,890 DEBUG [RS:0;fadb2fe0ba32:40681 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fadb2fe0ba32,40681,1732422576481'
2024-11-24T04:29:36,890 DEBUG [RS:0;fadb2fe0ba32:40681 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-24T04:29:36,890 DEBUG [RS:0;fadb2fe0ba32:40681 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-24T04:29:36,891 DEBUG [RS:0;fadb2fe0ba32:40681 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-24T04:29:36,891 INFO  [RS:0;fadb2fe0ba32:40681 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-24T04:29:36,891 INFO  [RS:0;fadb2fe0ba32:40681 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-24T04:29:36,940 WARN  [fadb2fe0ba32:35377 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-11-24T04:29:36,994 INFO  [RS:0;fadb2fe0ba32:40681 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fadb2fe0ba32%2C40681%2C1732422576481, suffix=, logDir=hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481, archiveDir=hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/oldWALs, maxLogs=32
2024-11-24T04:29:36,995 INFO  [RS:0;fadb2fe0ba32:40681 {}] monitor.StreamSlowMonitor(122): New stream slow monitor fadb2fe0ba32%2C40681%2C1732422576481.1732422576995
2024-11-24T04:29:37,003 INFO  [RS:0;fadb2fe0ba32:40681 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.1732422576995
2024-11-24T04:29:37,010 DEBUG [RS:0;fadb2fe0ba32:40681 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46579:46579),(127.0.0.1/127.0.0.1:41079:41079)]
2024-11-24T04:29:37,040 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-24T04:29:37,045 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:29:37,058 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:29:37,061 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:29:37,061 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:29:37,191 DEBUG [fadb2fe0ba32:35377 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-24T04:29:37,191 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=fadb2fe0ba32,40681,1732422576481
2024-11-24T04:29:37,193 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fadb2fe0ba32,40681,1732422576481, state=OPENING
2024-11-24T04:29:37,195 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-24T04:29:37,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35377-0x101483c93a00000, quorum=127.0.0.1:62127, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:29:37,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40681-0x101483c93a00001, quorum=127.0.0.1:62127, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:29:37,197 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-24T04:29:37,197 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-24T04:29:37,197 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-24T04:29:37,197 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=fadb2fe0ba32,40681,1732422576481}]
2024-11-24T04:29:37,351 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-24T04:29:37,353 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40669, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-24T04:29:37,357 INFO  [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-24T04:29:37,358 INFO  [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-24T04:29:37,360 INFO  [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fadb2fe0ba32%2C40681%2C1732422576481.meta, suffix=.meta, logDir=hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481, archiveDir=hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/oldWALs, maxLogs=32
2024-11-24T04:29:37,362 INFO  [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
2024-11-24T04:29:37,369 INFO  [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
2024-11-24T04:29:37,380 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46579:46579),(127.0.0.1/127.0.0.1:41079:41079)]
2024-11-24T04:29:37,383 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-24T04:29:37,383 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-24T04:29:37,384 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-24T04:29:37,384 INFO  [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-24T04:29:37,384 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-24T04:29:37,384 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T04:29:37,384 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-24T04:29:37,384 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-24T04:29:37,386 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-24T04:29:37,387 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-24T04:29:37,387 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:29:37,388 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T04:29:37,388 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-24T04:29:37,389 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-24T04:29:37,389 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:29:37,389 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T04:29:37,390 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-24T04:29:37,390 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-24T04:29:37,390 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:29:37,391 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T04:29:37,391 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-24T04:29:37,392 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-24T04:29:37,392 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:29:37,393 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T04:29:37,393 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-24T04:29:37,394 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/hbase/meta/1588230740
2024-11-24T04:29:37,395 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/hbase/meta/1588230740
2024-11-24T04:29:37,396 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-24T04:29:37,396 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-24T04:29:37,397 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-24T04:29:37,399 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-24T04:29:37,399 INFO  [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=873372, jitterRate=0.1105504035949707}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-24T04:29:37,400 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-24T04:29:37,401 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1732422577384Writing region info on filesystem at 1732422577384Initializing all the Stores at 1732422577385 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732422577385Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732422577386 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422577386Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732422577386Cleaning up temporary data from old regions at 1732422577396 (+10 ms)Running coprocessor post-open hooks at 1732422577400 (+4 ms)Region opened successfully at 1732422577400
2024-11-24T04:29:37,402 INFO  [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732422577351
2024-11-24T04:29:37,405 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-24T04:29:37,405 INFO  [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-24T04:29:37,406 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=fadb2fe0ba32,40681,1732422576481
2024-11-24T04:29:37,407 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fadb2fe0ba32,40681,1732422576481, state=OPEN
2024-11-24T04:29:37,415 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40681-0x101483c93a00001, quorum=127.0.0.1:62127, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-24T04:29:37,415 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35377-0x101483c93a00000, quorum=127.0.0.1:62127, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-24T04:29:37,415 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-24T04:29:37,415 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=fadb2fe0ba32,40681,1732422576481
2024-11-24T04:29:37,415 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-24T04:29:37,419 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-24T04:29:37,419 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fadb2fe0ba32,40681,1732422576481 in 218 msec
2024-11-24T04:29:37,422 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-24T04:29:37,422 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 632 msec
2024-11-24T04:29:37,423 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-24T04:29:37,423 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-24T04:29:37,425 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-24T04:29:37,425 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fadb2fe0ba32,40681,1732422576481, seqNum=-1]
2024-11-24T04:29:37,425 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-24T04:29:37,426 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37509, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-24T04:29:37,432 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 697 msec
2024-11-24T04:29:37,432 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732422577432, completionTime=-1
2024-11-24T04:29:37,433 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-24T04:29:37,433 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-24T04:29:37,435 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-24T04:29:37,435 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732422637435
2024-11-24T04:29:37,435 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732422697435
2024-11-24T04:29:37,435 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec
2024-11-24T04:29:37,435 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,35377,1732422576418-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-24T04:29:37,435 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,35377,1732422576418-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T04:29:37,435 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,35377,1732422576418-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T04:29:37,435 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-fadb2fe0ba32:35377, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T04:29:37,435 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-24T04:29:37,436 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-24T04:29:37,437 DEBUG [master/fadb2fe0ba32:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-24T04:29:37,440 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.919sec
2024-11-24T04:29:37,440 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-24T04:29:37,440 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-24T04:29:37,440 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-24T04:29:37,440 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-24T04:29:37,440 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-24T04:29:37,440 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,35377,1732422576418-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-24T04:29:37,440 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,35377,1732422576418-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-24T04:29:37,443 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-24T04:29:37,443 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-24T04:29:37,443 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,35377,1732422576418-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T04:29:37,500 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cd7b3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-24T04:29:37,500 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fadb2fe0ba32,35377,-1 for getting cluster id
2024-11-24T04:29:37,501 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-24T04:29:37,503 DEBUG [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4e7a7841-646c-4aea-b78c-b3202165c79a'
2024-11-24T04:29:37,503 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-24T04:29:37,503 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4e7a7841-646c-4aea-b78c-b3202165c79a"

2024-11-24T04:29:37,504 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d65fa4a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-24T04:29:37,504 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fadb2fe0ba32,35377,-1]
2024-11-24T04:29:37,504 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-24T04:29:37,504 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T04:29:37,506 INFO  [HMaster-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52956, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-24T04:29:37,507 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@abbe752, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-24T04:29:37,507 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-24T04:29:37,508 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fadb2fe0ba32,40681,1732422576481, seqNum=-1]
2024-11-24T04:29:37,508 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-24T04:29:37,510 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51556, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-24T04:29:37,512 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=fadb2fe0ba32,35377,1732422576418
2024-11-24T04:29:37,512 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T04:29:37,515 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-24T04:29:37,563 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/fadb2fe0ba32:0 server-side Connection retries=45
2024-11-24T04:29:37,563 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-24T04:29:37,563 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-24T04:29:37,563 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-24T04:29:37,563 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-24T04:29:37,563 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-24T04:29:37,563 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-24T04:29:37,563 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-24T04:29:37,564 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44377
2024-11-24T04:29:37,565 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44377 connecting to ZooKeeper ensemble=127.0.0.1:62127
2024-11-24T04:29:37,566 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T04:29:37,568 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T04:29:37,572 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:443770x0, quorum=127.0.0.1:62127, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-24T04:29:37,572 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:44377-0x101483c93a00002, quorum=127.0.0.1:62127, baseZNode=/hbase Set watcher on existing znode=/hbase/running
2024-11-24T04:29:37,572 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44377-0x101483c93a00002 connected
2024-11-24T04:29:37,572 DEBUG [pool-381-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT
2024-11-24T04:29:37,573 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-24T04:29:37,574 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-24T04:29:37,574 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:44377-0x101483c93a00002, quorum=127.0.0.1:62127, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-24T04:29:37,576 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44377-0x101483c93a00002, quorum=127.0.0.1:62127, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-24T04:29:37,576 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44377
2024-11-24T04:29:37,576 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44377
2024-11-24T04:29:37,577 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44377
2024-11-24T04:29:37,577 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44377
2024-11-24T04:29:37,577 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44377
2024-11-24T04:29:37,578 INFO  [RS:1;fadb2fe0ba32:44377 {}] regionserver.HRegionServer(746): ClusterId : 4e7a7841-646c-4aea-b78c-b3202165c79a
2024-11-24T04:29:37,578 DEBUG [RS:1;fadb2fe0ba32:44377 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-24T04:29:37,580 DEBUG [RS:1;fadb2fe0ba32:44377 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-24T04:29:37,580 DEBUG [RS:1;fadb2fe0ba32:44377 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-24T04:29:37,583 DEBUG [RS:1;fadb2fe0ba32:44377 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-24T04:29:37,584 DEBUG [RS:1;fadb2fe0ba32:44377 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c027df9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fadb2fe0ba32/172.17.0.2:0
2024-11-24T04:29:37,596 DEBUG [RS:1;fadb2fe0ba32:44377 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;fadb2fe0ba32:44377
2024-11-24T04:29:37,596 INFO  [RS:1;fadb2fe0ba32:44377 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-24T04:29:37,596 INFO  [RS:1;fadb2fe0ba32:44377 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-24T04:29:37,596 DEBUG [RS:1;fadb2fe0ba32:44377 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-24T04:29:37,597 INFO  [RS:1;fadb2fe0ba32:44377 {}] regionserver.HRegionServer(2659): reportForDuty to master=fadb2fe0ba32,35377,1732422576418 with port=44377, startcode=1732422577562
2024-11-24T04:29:37,597 DEBUG [RS:1;fadb2fe0ba32:44377 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-24T04:29:37,599 INFO  [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44329, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.3 (auth:SIMPLE), service=RegionServerStatusService
2024-11-24T04:29:37,599 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35377 {}] master.ServerManager(363): Checking decommissioned status of RegionServer fadb2fe0ba32,44377,1732422577562
2024-11-24T04:29:37,599 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35377 {}] master.ServerManager(517): Registering regionserver=fadb2fe0ba32,44377,1732422577562
2024-11-24T04:29:37,601 DEBUG [RS:1;fadb2fe0ba32:44377 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc
2024-11-24T04:29:37,601 DEBUG [RS:1;fadb2fe0ba32:44377 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40187
2024-11-24T04:29:37,601 DEBUG [RS:1;fadb2fe0ba32:44377 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-24T04:29:37,603 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35377-0x101483c93a00000, quorum=127.0.0.1:62127, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-24T04:29:37,603 DEBUG [RS:1;fadb2fe0ba32:44377 {}] zookeeper.ZKUtil(111): regionserver:44377-0x101483c93a00002, quorum=127.0.0.1:62127, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/fadb2fe0ba32,44377,1732422577562
2024-11-24T04:29:37,603 WARN  [RS:1;fadb2fe0ba32:44377 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-24T04:29:37,603 INFO  [RS:1;fadb2fe0ba32:44377 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-24T04:29:37,603 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [fadb2fe0ba32,44377,1732422577562]
2024-11-24T04:29:37,603 DEBUG [RS:1;fadb2fe0ba32:44377 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562
2024-11-24T04:29:37,607 INFO  [RS:1;fadb2fe0ba32:44377 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-24T04:29:37,608 INFO  [RS:1;fadb2fe0ba32:44377 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-24T04:29:37,609 INFO  [RS:1;fadb2fe0ba32:44377 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-24T04:29:37,609 INFO  [RS:1;fadb2fe0ba32:44377 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-24T04:29:37,609 INFO  [RS:1;fadb2fe0ba32:44377 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-24T04:29:37,610 INFO  [RS:1;fadb2fe0ba32:44377 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-24T04:29:37,610 INFO  [RS:1;fadb2fe0ba32:44377 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-24T04:29:37,610 DEBUG [RS:1;fadb2fe0ba32:44377 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:29:37,610 DEBUG [RS:1;fadb2fe0ba32:44377 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:29:37,610 DEBUG [RS:1;fadb2fe0ba32:44377 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:29:37,610 DEBUG [RS:1;fadb2fe0ba32:44377 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:29:37,610 DEBUG [RS:1;fadb2fe0ba32:44377 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:29:37,610 DEBUG [RS:1;fadb2fe0ba32:44377 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/fadb2fe0ba32:0, corePoolSize=2, maxPoolSize=2
2024-11-24T04:29:37,610 DEBUG [RS:1;fadb2fe0ba32:44377 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:29:37,611 DEBUG [RS:1;fadb2fe0ba32:44377 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:29:37,611 DEBUG [RS:1;fadb2fe0ba32:44377 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:29:37,611 DEBUG [RS:1;fadb2fe0ba32:44377 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:29:37,611 DEBUG [RS:1;fadb2fe0ba32:44377 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:29:37,611 DEBUG [RS:1;fadb2fe0ba32:44377 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:29:37,611 DEBUG [RS:1;fadb2fe0ba32:44377 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/fadb2fe0ba32:0, corePoolSize=3, maxPoolSize=3
2024-11-24T04:29:37,611 DEBUG [RS:1;fadb2fe0ba32:44377 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/fadb2fe0ba32:0, corePoolSize=3, maxPoolSize=3
2024-11-24T04:29:37,611 INFO  [RS:1;fadb2fe0ba32:44377 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-24T04:29:37,611 INFO  [RS:1;fadb2fe0ba32:44377 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-24T04:29:37,611 INFO  [RS:1;fadb2fe0ba32:44377 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-24T04:29:37,612 INFO  [RS:1;fadb2fe0ba32:44377 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-24T04:29:37,612 INFO  [RS:1;fadb2fe0ba32:44377 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-24T04:29:37,612 INFO  [RS:1;fadb2fe0ba32:44377 {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,44377,1732422577562-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-24T04:29:37,628 INFO  [RS:1;fadb2fe0ba32:44377 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-24T04:29:37,628 INFO  [RS:1;fadb2fe0ba32:44377 {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,44377,1732422577562-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-24T04:29:37,629 INFO  [RS:1;fadb2fe0ba32:44377 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T04:29:37,629 INFO  [RS:1;fadb2fe0ba32:44377 {}] regionserver.Replication(171): fadb2fe0ba32,44377,1732422577562 started
2024-11-24T04:29:37,643 INFO  [RS:1;fadb2fe0ba32:44377 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T04:29:37,643 INFO  [RS:1;fadb2fe0ba32:44377 {}] regionserver.HRegionServer(1482): Serving as fadb2fe0ba32,44377,1732422577562, RpcServer on fadb2fe0ba32/172.17.0.2:44377, sessionid=0x101483c93a00002
2024-11-24T04:29:37,643 DEBUG [RS:1;fadb2fe0ba32:44377 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-24T04:29:37,643 DEBUG [RS:1;fadb2fe0ba32:44377 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager fadb2fe0ba32,44377,1732422577562
2024-11-24T04:29:37,643 DEBUG [RS:1;fadb2fe0ba32:44377 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fadb2fe0ba32,44377,1732422577562'
2024-11-24T04:29:37,643 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(2882): Started new server=Thread[RS:1;fadb2fe0ba32:44377,5,FailOnTimeoutGroup]
2024-11-24T04:29:37,644 DEBUG [RS:1;fadb2fe0ba32:44377 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-24T04:29:37,644 INFO  [Time-limited test {}] wal.TestLogRolling(207): Replication=2
2024-11-24T04:29:37,644 DEBUG [RS:1;fadb2fe0ba32:44377 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-24T04:29:37,644 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-24T04:29:37,645 DEBUG [RS:1;fadb2fe0ba32:44377 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-24T04:29:37,645 DEBUG [RS:1;fadb2fe0ba32:44377 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-24T04:29:37,645 DEBUG [RS:1;fadb2fe0ba32:44377 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager fadb2fe0ba32,44377,1732422577562
2024-11-24T04:29:37,645 DEBUG [RS:1;fadb2fe0ba32:44377 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fadb2fe0ba32,44377,1732422577562'
2024-11-24T04:29:37,645 DEBUG [RS:1;fadb2fe0ba32:44377 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-24T04:29:37,645 DEBUG [RS:1;fadb2fe0ba32:44377 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-24T04:29:37,645 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncConnectionImpl(321): The fetched master address is fadb2fe0ba32,35377,1732422576418
2024-11-24T04:29:37,645 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@cb496b3
2024-11-24T04:29:37,646 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-24T04:29:37,646 DEBUG [RS:1;fadb2fe0ba32:44377 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-24T04:29:37,646 INFO  [RS:1;fadb2fe0ba32:44377 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-24T04:29:37,646 INFO  [RS:1;fadb2fe0ba32:44377 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-24T04:29:37,648 INFO  [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52972, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-24T04:29:37,648 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35377 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-24T04:29:37,648 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35377 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-24T04:29:37,649 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35377 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRollOnDatanodeDeath', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-24T04:29:37,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35377 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath
2024-11-24T04:29:37,652 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_PRE_OPERATION
2024-11-24T04:29:37,652 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:29:37,652 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35377 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRollOnDatanodeDeath"
 procId is: 4
2024-11-24T04:29:37,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-24T04:29:37,653 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-24T04:29:37,661 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46879 is added to blk_1073741835_1011 (size=393)
2024-11-24T04:29:37,661 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37029 is added to blk_1073741835_1011 (size=393)
2024-11-24T04:29:37,663 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0ce341802081e9b50470c63603e22c62, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1732422577648.0ce341802081e9b50470c63603e22c62.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnDatanodeDeath', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc
2024-11-24T04:29:37,668 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46879 is added to blk_1073741836_1012 (size=76)
2024-11-24T04:29:37,669 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37029 is added to blk_1073741836_1012 (size=76)
2024-11-24T04:29:37,669 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1732422577648.0ce341802081e9b50470c63603e22c62.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T04:29:37,669 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1722): Closing 0ce341802081e9b50470c63603e22c62, disabling compactions & flushes
2024-11-24T04:29:37,670 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1732422577648.0ce341802081e9b50470c63603e22c62.
2024-11-24T04:29:37,670 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1732422577648.0ce341802081e9b50470c63603e22c62.
2024-11-24T04:29:37,670 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1732422577648.0ce341802081e9b50470c63603e22c62. after waiting 0 ms
2024-11-24T04:29:37,670 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1732422577648.0ce341802081e9b50470c63603e22c62.
2024-11-24T04:29:37,670 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1732422577648.0ce341802081e9b50470c63603e22c62.
2024-11-24T04:29:37,670 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0ce341802081e9b50470c63603e22c62:
Waiting for close lock at 1732422577669Disabling compacts and flushes for region at 1732422577669Disabling writes for close at 1732422577670 (+1 ms)Writing region close event to WAL at 1732422577670Closed at 1732422577670
2024-11-24T04:29:37,672 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ADD_TO_META
2024-11-24T04:29:37,672 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnDatanodeDeath,,1732422577648.0ce341802081e9b50470c63603e22c62.","families":{"info":[{"qualifier":"regioninfo","vlen":75,"tag":[],"timestamp":"1732422577672"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732422577672"}]},"ts":"1732422577672"}
2024-11-24T04:29:37,675 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-24T04:29:37,676 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-24T04:29:37,676 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732422577676"}]},"ts":"1732422577676"}
2024-11-24T04:29:37,678 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLING in hbase:meta
2024-11-24T04:29:37,679 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=0ce341802081e9b50470c63603e22c62, ASSIGN}]
2024-11-24T04:29:37,680 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=0ce341802081e9b50470c63603e22c62, ASSIGN
2024-11-24T04:29:37,681 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=0ce341802081e9b50470c63603e22c62, ASSIGN; state=OFFLINE, location=fadb2fe0ba32,40681,1732422576481; forceNewPlan=false, retain=false
2024-11-24T04:29:37,748 INFO  [RS:1;fadb2fe0ba32:44377 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fadb2fe0ba32%2C44377%2C1732422577562, suffix=, logDir=hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562, archiveDir=hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/oldWALs, maxLogs=32
2024-11-24T04:29:37,749 INFO  [RS:1;fadb2fe0ba32:44377 {}] monitor.StreamSlowMonitor(122): New stream slow monitor fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
2024-11-24T04:29:37,756 INFO  [RS:1;fadb2fe0ba32:44377 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
2024-11-24T04:29:37,757 DEBUG [RS:1;fadb2fe0ba32:44377 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46579:46579),(127.0.0.1/127.0.0.1:41079:41079)]
2024-11-24T04:29:37,832 INFO  [fadb2fe0ba32:35377 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 
2024-11-24T04:29:37,833 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=0ce341802081e9b50470c63603e22c62, regionState=OPENING, regionLocation=fadb2fe0ba32,40681,1732422576481
2024-11-24T04:29:37,836 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=0ce341802081e9b50470c63603e22c62, ASSIGN because future has completed
2024-11-24T04:29:37,836 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0ce341802081e9b50470c63603e22c62, server=fadb2fe0ba32,40681,1732422576481}]
2024-11-24T04:29:37,993 INFO  [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRollOnDatanodeDeath,,1732422577648.0ce341802081e9b50470c63603e22c62.
2024-11-24T04:29:37,994 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 0ce341802081e9b50470c63603e22c62, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1732422577648.0ce341802081e9b50470c63603e22c62.', STARTKEY => '', ENDKEY => ''}
2024-11-24T04:29:37,994 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnDatanodeDeath 0ce341802081e9b50470c63603e22c62
2024-11-24T04:29:37,994 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1732422577648.0ce341802081e9b50470c63603e22c62.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T04:29:37,994 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 0ce341802081e9b50470c63603e22c62
2024-11-24T04:29:37,994 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 0ce341802081e9b50470c63603e22c62
2024-11-24T04:29:37,996 INFO  [StoreOpener-0ce341802081e9b50470c63603e22c62-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 0ce341802081e9b50470c63603e22c62 
2024-11-24T04:29:37,997 INFO  [StoreOpener-0ce341802081e9b50470c63603e22c62-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0ce341802081e9b50470c63603e22c62 columnFamilyName info
2024-11-24T04:29:37,997 DEBUG [StoreOpener-0ce341802081e9b50470c63603e22c62-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:29:37,998 INFO  [StoreOpener-0ce341802081e9b50470c63603e22c62-1 {}] regionserver.HStore(327): Store=0ce341802081e9b50470c63603e22c62/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-24T04:29:37,998 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 0ce341802081e9b50470c63603e22c62
2024-11-24T04:29:37,999 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62
2024-11-24T04:29:37,999 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62
2024-11-24T04:29:37,999 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 0ce341802081e9b50470c63603e22c62
2024-11-24T04:29:38,000 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 0ce341802081e9b50470c63603e22c62
2024-11-24T04:29:38,001 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 0ce341802081e9b50470c63603e22c62
2024-11-24T04:29:38,003 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-24T04:29:38,004 INFO  [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 0ce341802081e9b50470c63603e22c62; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=829993, jitterRate=0.055390894412994385}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-24T04:29:38,004 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0ce341802081e9b50470c63603e22c62
2024-11-24T04:29:38,004 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 0ce341802081e9b50470c63603e22c62:
Running coprocessor pre-open hook at 1732422577995Writing region info on filesystem at 1732422577995Initializing all the Stores at 1732422577995Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422577996 (+1 ms)Cleaning up temporary data from old regions at 1732422578000 (+4 ms)Running coprocessor post-open hooks at 1732422578004 (+4 ms)Region opened successfully at 1732422578004
2024-11-24T04:29:38,006 INFO  [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRollOnDatanodeDeath,,1732422577648.0ce341802081e9b50470c63603e22c62., pid=6, masterSystemTime=1732422577989
2024-11-24T04:29:38,008 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRollOnDatanodeDeath,,1732422577648.0ce341802081e9b50470c63603e22c62.
2024-11-24T04:29:38,008 INFO  [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRollOnDatanodeDeath,,1732422577648.0ce341802081e9b50470c63603e22c62.
2024-11-24T04:29:38,009 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=0ce341802081e9b50470c63603e22c62, regionState=OPEN, openSeqNum=2, regionLocation=fadb2fe0ba32,40681,1732422576481
2024-11-24T04:29:38,011 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0ce341802081e9b50470c63603e22c62, server=fadb2fe0ba32,40681,1732422576481 because future has completed
2024-11-24T04:29:38,016 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-24T04:29:38,016 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 0ce341802081e9b50470c63603e22c62, server=fadb2fe0ba32,40681,1732422576481 in 177 msec
2024-11-24T04:29:38,019 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-24T04:29:38,019 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=0ce341802081e9b50470c63603e22c62, ASSIGN in 337 msec
2024-11-24T04:29:38,020 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-24T04:29:38,020 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732422578020"}]},"ts":"1732422578020"}
2024-11-24T04:29:38,022 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLED in hbase:meta
2024-11-24T04:29:38,023 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_POST_OPERATION
2024-11-24T04:29:38,025 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath in 374 msec
2024-11-24T04:29:42,886 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-24T04:29:42,888 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:29:42,899 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:29:42,901 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:29:42,901 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:29:42,908 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnDatanodeDeath'
2024-11-24T04:29:46,266 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-24T04:29:46,266 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-11-24T04:29:46,267 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath
2024-11-24T04:29:46,267 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath Metrics about Tables on a single HBase RegionServer
2024-11-24T04:29:46,268 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-24T04:29:46,268 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-11-24T04:29:46,268 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-24T04:29:46,268 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-11-24T04:29:47,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-24T04:29:47,743 INFO  [RPCClient-NioEventLoopGroup-4-11 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnDatanodeDeath completed
2024-11-24T04:29:47,743 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRollOnDatanodeDeath,, stopping at row=TestLogRolling-testLogRollOnDatanodeDeath ,, for max=2147483647 with caching=100
2024-11-24T04:29:47,746 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRollOnDatanodeDeath
2024-11-24T04:29:47,746 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRollOnDatanodeDeath,,1732422577648.0ce341802081e9b50470c63603e22c62.
2024-11-24T04:29:47,760 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T04:29:47,763 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-24T04:29:47,764 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-24T04:29:47,764 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-24T04:29:47,764 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-24T04:29:47,764 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@36d0b5ff{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/hadoop.log.dir/,AVAILABLE}
2024-11-24T04:29:47,765 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@51f59516{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-24T04:29:47,881 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3c5438f9{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/java.io.tmpdir/jetty-localhost-33125-hadoop-hdfs-3_4_1-tests_jar-_-any-2262121710339294393/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T04:29:47,881 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2e7a8425{HTTP/1.1, (http/1.1)}{localhost:33125}
2024-11-24T04:29:47,881 INFO  [Time-limited test {}] server.Server(415): Started @116943ms
2024-11-24T04:29:47,883 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-24T04:29:47,952 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T04:29:47,956 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-24T04:29:47,957 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-24T04:29:47,957 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-24T04:29:47,957 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-24T04:29:47,958 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@712f5f14{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/hadoop.log.dir/,AVAILABLE}
2024-11-24T04:29:47,958 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3afd309b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-24T04:29:48,012 WARN  [Thread-829 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/cluster_56272640-4b45-3980-6187-aab4c21513ae/data/data5/current/BP-1909738581-172.17.0.2-1732422575571/current, will proceed with Du for space computation calculation, 
2024-11-24T04:29:48,012 WARN  [Thread-830 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/cluster_56272640-4b45-3980-6187-aab4c21513ae/data/data6/current/BP-1909738581-172.17.0.2-1732422575571/current, will proceed with Du for space computation calculation, 
2024-11-24T04:29:48,030 WARN  [Thread-809 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-24T04:29:48,034 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd3de3c29a5495af7 with lease ID 0xbac364592180ee86: Processing first storage report for DS-4844294c-4a9d-4087-a05b-bdb1c1eed91d from datanode DatanodeRegistration(127.0.0.1:35379, datanodeUuid=1fddb4ec-42cd-40c2-8e6d-fb70aba59f5b, infoPort=39949, infoSecurePort=0, ipcPort=43999, storageInfo=lv=-57;cid=testClusterID;nsid=612618959;c=1732422575571)
2024-11-24T04:29:48,034 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd3de3c29a5495af7 with lease ID 0xbac364592180ee86: from storage DS-4844294c-4a9d-4087-a05b-bdb1c1eed91d node DatanodeRegistration(127.0.0.1:35379, datanodeUuid=1fddb4ec-42cd-40c2-8e6d-fb70aba59f5b, infoPort=39949, infoSecurePort=0, ipcPort=43999, storageInfo=lv=-57;cid=testClusterID;nsid=612618959;c=1732422575571), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T04:29:48,034 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd3de3c29a5495af7 with lease ID 0xbac364592180ee86: Processing first storage report for DS-0c7b6cd9-a035-4654-b0a1-7ecb98050d38 from datanode DatanodeRegistration(127.0.0.1:35379, datanodeUuid=1fddb4ec-42cd-40c2-8e6d-fb70aba59f5b, infoPort=39949, infoSecurePort=0, ipcPort=43999, storageInfo=lv=-57;cid=testClusterID;nsid=612618959;c=1732422575571)
2024-11-24T04:29:48,034 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd3de3c29a5495af7 with lease ID 0xbac364592180ee86: from storage DS-0c7b6cd9-a035-4654-b0a1-7ecb98050d38 node DatanodeRegistration(127.0.0.1:35379, datanodeUuid=1fddb4ec-42cd-40c2-8e6d-fb70aba59f5b, infoPort=39949, infoSecurePort=0, ipcPort=43999, storageInfo=lv=-57;cid=testClusterID;nsid=612618959;c=1732422575571), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T04:29:48,102 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@39835cdd{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/java.io.tmpdir/jetty-localhost-40445-hadoop-hdfs-3_4_1-tests_jar-_-any-16132302522723797814/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T04:29:48,102 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@12b19204{HTTP/1.1, (http/1.1)}{localhost:40445}
2024-11-24T04:29:48,102 INFO  [Time-limited test {}] server.Server(415): Started @117164ms
2024-11-24T04:29:48,104 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-24T04:29:48,143 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T04:29:48,149 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-24T04:29:48,151 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-24T04:29:48,151 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-24T04:29:48,151 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-24T04:29:48,154 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@68004957{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/hadoop.log.dir/,AVAILABLE}
2024-11-24T04:29:48,154 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@78e445ac{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-24T04:29:48,210 WARN  [Thread-864 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/cluster_56272640-4b45-3980-6187-aab4c21513ae/data/data7/current/BP-1909738581-172.17.0.2-1732422575571/current, will proceed with Du for space computation calculation, 
2024-11-24T04:29:48,210 WARN  [Thread-865 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/cluster_56272640-4b45-3980-6187-aab4c21513ae/data/data8/current/BP-1909738581-172.17.0.2-1732422575571/current, will proceed with Du for space computation calculation, 
2024-11-24T04:29:48,233 WARN  [Thread-844 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-24T04:29:48,236 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1d94394389840ffd with lease ID 0xbac364592180ee87: Processing first storage report for DS-535553ae-5a1c-418e-828a-fe05e393ffd4 from datanode DatanodeRegistration(127.0.0.1:36375, datanodeUuid=4b89cc0c-a856-401b-8bf0-9a7a3e1a348c, infoPort=35349, infoSecurePort=0, ipcPort=45759, storageInfo=lv=-57;cid=testClusterID;nsid=612618959;c=1732422575571)
2024-11-24T04:29:48,236 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1d94394389840ffd with lease ID 0xbac364592180ee87: from storage DS-535553ae-5a1c-418e-828a-fe05e393ffd4 node DatanodeRegistration(127.0.0.1:36375, datanodeUuid=4b89cc0c-a856-401b-8bf0-9a7a3e1a348c, infoPort=35349, infoSecurePort=0, ipcPort=45759, storageInfo=lv=-57;cid=testClusterID;nsid=612618959;c=1732422575571), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T04:29:48,237 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1d94394389840ffd with lease ID 0xbac364592180ee87: Processing first storage report for DS-31ec27bb-57c5-49d7-8dd0-e1ddc016a1b7 from datanode DatanodeRegistration(127.0.0.1:36375, datanodeUuid=4b89cc0c-a856-401b-8bf0-9a7a3e1a348c, infoPort=35349, infoSecurePort=0, ipcPort=45759, storageInfo=lv=-57;cid=testClusterID;nsid=612618959;c=1732422575571)
2024-11-24T04:29:48,237 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1d94394389840ffd with lease ID 0xbac364592180ee87: from storage DS-31ec27bb-57c5-49d7-8dd0-e1ddc016a1b7 node DatanodeRegistration(127.0.0.1:36375, datanodeUuid=4b89cc0c-a856-401b-8bf0-9a7a3e1a348c, infoPort=35349, infoSecurePort=0, ipcPort=45759, storageInfo=lv=-57;cid=testClusterID;nsid=612618959;c=1732422575571), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-24T04:29:48,288 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5ea37f0d{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/java.io.tmpdir/jetty-localhost-39547-hadoop-hdfs-3_4_1-tests_jar-_-any-17483285415299863747/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T04:29:48,289 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@15010086{HTTP/1.1, (http/1.1)}{localhost:39547}
2024-11-24T04:29:48,289 INFO  [Time-limited test {}] server.Server(415): Started @117350ms
2024-11-24T04:29:48,290 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-24T04:29:48,394 WARN  [Thread-890 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/cluster_56272640-4b45-3980-6187-aab4c21513ae/data/data9/current/BP-1909738581-172.17.0.2-1732422575571/current, will proceed with Du for space computation calculation, 
2024-11-24T04:29:48,395 WARN  [Thread-891 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/cluster_56272640-4b45-3980-6187-aab4c21513ae/data/data10/current/BP-1909738581-172.17.0.2-1732422575571/current, will proceed with Du for space computation calculation, 
2024-11-24T04:29:48,417 WARN  [Thread-879 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-24T04:29:48,421 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe076e6d9ede40fac with lease ID 0xbac364592180ee88: Processing first storage report for DS-48952f18-74aa-43bd-ad38-e5a4deb9635a from datanode DatanodeRegistration(127.0.0.1:45685, datanodeUuid=a280f544-a74c-41cc-82b9-31a5022433e7, infoPort=33121, infoSecurePort=0, ipcPort=39475, storageInfo=lv=-57;cid=testClusterID;nsid=612618959;c=1732422575571)
2024-11-24T04:29:48,421 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe076e6d9ede40fac with lease ID 0xbac364592180ee88: from storage DS-48952f18-74aa-43bd-ad38-e5a4deb9635a node DatanodeRegistration(127.0.0.1:45685, datanodeUuid=a280f544-a74c-41cc-82b9-31a5022433e7, infoPort=33121, infoSecurePort=0, ipcPort=39475, storageInfo=lv=-57;cid=testClusterID;nsid=612618959;c=1732422575571), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T04:29:48,421 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe076e6d9ede40fac with lease ID 0xbac364592180ee88: Processing first storage report for DS-13973aed-617b-4ff7-b034-f5ed76e88fcc from datanode DatanodeRegistration(127.0.0.1:45685, datanodeUuid=a280f544-a74c-41cc-82b9-31a5022433e7, infoPort=33121, infoSecurePort=0, ipcPort=39475, storageInfo=lv=-57;cid=testClusterID;nsid=612618959;c=1732422575571)
2024-11-24T04:29:48,421 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe076e6d9ede40fac with lease ID 0xbac364592180ee88: from storage DS-13973aed-617b-4ff7-b034-f5ed76e88fcc node DatanodeRegistration(127.0.0.1:45685, datanodeUuid=a280f544-a74c-41cc-82b9-31a5022433e7, infoPort=33121, infoSecurePort=0, ipcPort=39475, storageInfo=lv=-57;cid=testClusterID;nsid=612618959;c=1732422575571), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T04:29:48,426 WARN  [ResponseProcessor for block BP-1909738581-172.17.0.2-1732422575571:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1909738581-172.17.0.2-1732422575571:blk_1073741834_1010
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:29:48,426 WARN  [ResponseProcessor for block BP-1909738581-172.17.0.2-1732422575571:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1909738581-172.17.0.2-1732422575571:blk_1073741833_1009
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:29:48,426 WARN  [ResponseProcessor for block BP-1909738581-172.17.0.2-1732422575571:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1909738581-172.17.0.2-1732422575571:blk_1073741830_1006
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:29:48,426 WARN  [ResponseProcessor for block BP-1909738581-172.17.0.2-1732422575571:blk_1073741837_1013 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1909738581-172.17.0.2-1732422575571:blk_1073741837_1013
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:29:48,427 WARN  [DataStreamer for file /user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta block BP-1909738581-172.17.0.2-1732422575571:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1909738581-172.17.0.2-1732422575571:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37029,DS-1c26aeaf-2106-4f04-a38e-568cd2360db7,DISK], DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37029,DS-1c26aeaf-2106-4f04-a38e-568cd2360db7,DISK]) is bad.
2024-11-24T04:29:48,427 WARN  [DataStreamer for file /user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.1732422576995 block BP-1909738581-172.17.0.2-1732422575571:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1909738581-172.17.0.2-1732422575571:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37029,DS-1c26aeaf-2106-4f04-a38e-568cd2360db7,DISK], DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37029,DS-1c26aeaf-2106-4f04-a38e-568cd2360db7,DISK]) is bad.
2024-11-24T04:29:48,427 WARN  [DataStreamer for file /user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/MasterData/WALs/fadb2fe0ba32,35377,1732422576418/fadb2fe0ba32%2C35377%2C1732422576418.1732422576614 block BP-1909738581-172.17.0.2-1732422575571:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1909738581-172.17.0.2-1732422575571:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37029,DS-1c26aeaf-2106-4f04-a38e-568cd2360db7,DISK], DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37029,DS-1c26aeaf-2106-4f04-a38e-568cd2360db7,DISK]) is bad.
2024-11-24T04:29:48,427 WARN  [DataStreamer for file /user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749 block BP-1909738581-172.17.0.2-1732422575571:blk_1073741837_1013 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1909738581-172.17.0.2-1732422575571:blk_1073741837_1013 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37029,DS-1c26aeaf-2106-4f04-a38e-568cd2360db7,DISK], DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37029,DS-1c26aeaf-2106-4f04-a38e-568cd2360db7,DISK]) is bad.
2024-11-24T04:29:48,429 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@21d5e4af{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T04:29:48,429 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-271356777_22 at /127.0.0.1:49910 [Receiving block BP-1909738581-172.17.0.2-1732422575571:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:37029:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:49910 dst: /127.0.0.1:37029
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T04:29:48,429 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_620784384_22 at /127.0.0.1:56422 [Receiving block BP-1909738581-172.17.0.2-1732422575571:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:46879:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:56422 dst: /127.0.0.1:46879
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T04:29:48,429 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_620784384_22 at /127.0.0.1:49934 [Receiving block BP-1909738581-172.17.0.2-1732422575571:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:37029:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:49934 dst: /127.0.0.1:37029
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T04:29:48,429 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@145c0180{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-24T04:29:48,430 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-24T04:29:48,429 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_620784384_22 at /127.0.0.1:56438 [Receiving block BP-1909738581-172.17.0.2-1732422575571:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:46879:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:56438 dst: /127.0.0.1:46879
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T04:29:48,429 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-271356777_22 at /127.0.0.1:56388 [Receiving block BP-1909738581-172.17.0.2-1732422575571:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:46879:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:56388 dst: /127.0.0.1:46879
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T04:29:48,429 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_620784384_22 at /127.0.0.1:49942 [Receiving block BP-1909738581-172.17.0.2-1732422575571:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:37029:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:49942 dst: /127.0.0.1:37029
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T04:29:48,430 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7720beab{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-24T04:29:48,430 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@69bbaec1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/hadoop.log.dir/,STOPPED}
2024-11-24T04:29:48,430 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1293781119_22 at /127.0.0.1:56456 [Receiving block BP-1909738581-172.17.0.2-1732422575571:blk_1073741837_1013] {}] datanode.DataXceiver(331): 127.0.0.1:46879:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:56456 dst: /127.0.0.1:46879
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T04:29:48,430 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1293781119_22 at /127.0.0.1:49972 [Receiving block BP-1909738581-172.17.0.2-1732422575571:blk_1073741837_1013] {}] datanode.DataXceiver(331): 127.0.0.1:37029:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:49972 dst: /127.0.0.1:37029
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T04:29:48,431 WARN  [BP-1909738581-172.17.0.2-1732422575571 heartbeating to localhost/127.0.0.1:40187 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-24T04:29:48,432 WARN  [BP-1909738581-172.17.0.2-1732422575571 heartbeating to localhost/127.0.0.1:40187 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1909738581-172.17.0.2-1732422575571 (Datanode Uuid e3eba06e-e494-45bd-a171-1b16a8edcca7) service to localhost/127.0.0.1:40187
2024-11-24T04:29:48,432 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-24T04:29:48,432 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-24T04:29:48,432 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/cluster_56272640-4b45-3980-6187-aab4c21513ae/data/data3/current/BP-1909738581-172.17.0.2-1732422575571 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T04:29:48,433 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/cluster_56272640-4b45-3980-6187-aab4c21513ae/data/data4/current/BP-1909738581-172.17.0.2-1732422575571 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T04:29:48,433 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-24T04:29:48,434 WARN  [DataStreamer for file /user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/MasterData/WALs/fadb2fe0ba32,35377,1732422576418/fadb2fe0ba32%2C35377%2C1732422576418.1732422576614 block BP-1909738581-172.17.0.2-1732422575571:blk_1073741830_1006 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741830_1006
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:29:48,434 WARN  [DataStreamer for file /user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta block BP-1909738581-172.17.0.2-1732422575571:blk_1073741834_1010 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741834_1010
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:29:48,435 WARN  [DataStreamer for file /user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.1732422576995 block BP-1909738581-172.17.0.2-1732422575571:blk_1073741833_1009 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741833_1009
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:29:48,435 WARN  [DataStreamer for file /user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749 block BP-1909738581-172.17.0.2-1732422575571:blk_1073741837_1013 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741837_1013
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:29:48,437 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@ab5393f{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T04:29:48,437 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@47f82e76{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-24T04:29:48,437 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-24T04:29:48,437 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@30e7c448{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-24T04:29:48,438 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@257cf4bb{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/hadoop.log.dir/,STOPPED}
2024-11-24T04:29:48,440 WARN  [BP-1909738581-172.17.0.2-1732422575571 heartbeating to localhost/127.0.0.1:40187 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-24T04:29:48,440 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-24T04:29:48,440 WARN  [BP-1909738581-172.17.0.2-1732422575571 heartbeating to localhost/127.0.0.1:40187 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1909738581-172.17.0.2-1732422575571 (Datanode Uuid 2fcf51d3-1c3d-4734-9f68-e9df11c392d4) service to localhost/127.0.0.1:40187
2024-11-24T04:29:48,440 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-24T04:29:48,440 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/cluster_56272640-4b45-3980-6187-aab4c21513ae/data/data1/current/BP-1909738581-172.17.0.2-1732422575571 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T04:29:48,440 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/cluster_56272640-4b45-3980-6187-aab4c21513ae/data/data2/current/BP-1909738581-172.17.0.2-1732422575571 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T04:29:48,441 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-24T04:29:48,445 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRollOnDatanodeDeath', row='row0002', locateType=CURRENT is [region=TestLogRolling-testLogRollOnDatanodeDeath,,1732422577648.0ce341802081e9b50470c63603e22c62., hostname=fadb2fe0ba32,40681,1732422576481, seqNum=2]
2024-11-24T04:29:48,446 ERROR [FSHLog-0-hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc-prefix:fadb2fe0ba32,40681,1732422576481 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:29:48,446 WARN  [FSHLog-0-hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc-prefix:fadb2fe0ba32,40681,1732422576481 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:29:48,447 INFO  [regionserver/fadb2fe0ba32:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:29:48,447 DEBUG [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog fadb2fe0ba32%2C40681%2C1732422576481:(num 1732422576995) roll requested
2024-11-24T04:29:48,447 INFO  [regionserver/fadb2fe0ba32:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor fadb2fe0ba32%2C40681%2C1732422576481.1732422588447
2024-11-24T04:29:48,449 WARN  [Thread-901 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741838_1018
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:29:48,450 WARN  [Thread-901 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1909738581-172.17.0.2-1732422575571:blk_1073741838_1018 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37029,DS-1c26aeaf-2106-4f04-a38e-568cd2360db7,DISK], DatanodeInfoWithStorage[127.0.0.1:36375,DS-535553ae-5a1c-418e-828a-fe05e393ffd4,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37029,DS-1c26aeaf-2106-4f04-a38e-568cd2360db7,DISK]) is bad.
2024-11-24T04:29:48,450 WARN  [Thread-901 {}] hdfs.DataStreamer(1850): Abandoning BP-1909738581-172.17.0.2-1732422575571:blk_1073741838_1018
2024-11-24T04:29:48,452 WARN  [Thread-901 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37029,DS-1c26aeaf-2106-4f04-a38e-568cd2360db7,DISK]
2024-11-24T04:29:48,455 WARN  [Thread-901 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741839_1019
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:29:48,455 WARN  [Thread-901 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1909738581-172.17.0.2-1732422575571:blk_1073741839_1019 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK], DatanodeInfoWithStorage[127.0.0.1:35379,DS-4844294c-4a9d-4087-a05b-bdb1c1eed91d,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]) is bad.
2024-11-24T04:29:48,455 WARN  [Thread-901 {}] hdfs.DataStreamer(1850): Abandoning BP-1909738581-172.17.0.2-1732422575571:blk_1073741839_1019
2024-11-24T04:29:48,455 WARN  [Thread-901 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]
2024-11-24T04:29:48,460 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:48,460 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:48,460 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:48,460 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:48,460 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:48,460 INFO  [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.1732422576995 with entries=1, filesize=455 B; new WAL /user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.1732422588447
2024-11-24T04:29:48,461 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:29:48,461 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:29:48,461 DEBUG [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35349:35349),(127.0.0.1/127.0.0.1:39949:39949)]
2024-11-24T04:29:48,461 DEBUG [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.1732422576995 is not closed yet, will try archiving it next time
2024-11-24T04:29:48,462 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils
2024-11-24T04:29:48,462 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease()
2024-11-24T04:29:48,462 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.1732422576995
2024-11-24T04:29:48,465 WARN  [IPC Server handler 4 on default port 40187 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.1732422576995 has not been closed. Lease recovery is in progress. RecoveryId = 1021 for block blk_1073741833_1009
2024-11-24T04:29:48,468 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.1732422576995 after 4ms
2024-11-24T04:29:48,586 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:29:49,612 INFO  [regionserver/fadb2fe0ba32:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:29:50,462 INFO  [regionserver/fadb2fe0ba32:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:29:50,463 INFO  [Time-limited test {}] wal.TestLogRolling(261): log.getCurrentFileName(): hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.1732422588447
2024-11-24T04:29:50,463 WARN  [ResponseProcessor for block BP-1909738581-172.17.0.2-1732422575571:blk_1073741840_1020 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1909738581-172.17.0.2-1732422575571:blk_1073741840_1020
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:29:50,464 WARN  [DataStreamer for file /user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.1732422588447 block BP-1909738581-172.17.0.2-1732422575571:blk_1073741840_1020 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1909738581-172.17.0.2-1732422575571:blk_1073741840_1020 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36375,DS-535553ae-5a1c-418e-828a-fe05e393ffd4,DISK], DatanodeInfoWithStorage[127.0.0.1:35379,DS-4844294c-4a9d-4087-a05b-bdb1c1eed91d,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36375,DS-535553ae-5a1c-418e-828a-fe05e393ffd4,DISK]) is bad.
2024-11-24T04:29:50,464 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_620784384_22 at /127.0.0.1:59792 [Receiving block BP-1909738581-172.17.0.2-1732422575571:blk_1073741840_1020] {}] datanode.DataXceiver(331): 127.0.0.1:36375:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:59792 dst: /127.0.0.1:36375
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T04:29:50,465 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_620784384_22 at /127.0.0.1:34850 [Receiving block BP-1909738581-172.17.0.2-1732422575571:blk_1073741840_1020] {}] datanode.DataXceiver(331): 127.0.0.1:35379:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:34850 dst: /127.0.0.1:35379
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T04:29:50,468 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@39835cdd{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T04:29:50,468 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@12b19204{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-24T04:29:50,468 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-24T04:29:50,468 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3afd309b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-24T04:29:50,469 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@712f5f14{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/hadoop.log.dir/,STOPPED}
2024-11-24T04:29:50,470 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-24T04:29:50,470 WARN  [BP-1909738581-172.17.0.2-1732422575571 heartbeating to localhost/127.0.0.1:40187 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-24T04:29:50,470 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-24T04:29:50,470 WARN  [BP-1909738581-172.17.0.2-1732422575571 heartbeating to localhost/127.0.0.1:40187 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1909738581-172.17.0.2-1732422575571 (Datanode Uuid 4b89cc0c-a856-401b-8bf0-9a7a3e1a348c) service to localhost/127.0.0.1:40187
2024-11-24T04:29:50,471 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/cluster_56272640-4b45-3980-6187-aab4c21513ae/data/data7/current/BP-1909738581-172.17.0.2-1732422575571 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T04:29:50,471 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/cluster_56272640-4b45-3980-6187-aab4c21513ae/data/data8/current/BP-1909738581-172.17.0.2-1732422575571 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T04:29:50,471 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-24T04:29:50,587 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:29:51,612 INFO  [regionserver/fadb2fe0ba32:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:29:52,462 INFO  [regionserver/fadb2fe0ba32:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:29:52,463 WARN  [regionserver/fadb2fe0ba32:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:35379,DS-4844294c-4a9d-4087-a05b-bdb1c1eed91d,DISK]]
2024-11-24T04:29:52,463 DEBUG [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog fadb2fe0ba32%2C40681%2C1732422576481:(num 1732422588447) roll requested
2024-11-24T04:29:52,463 INFO  [regionserver/fadb2fe0ba32:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor fadb2fe0ba32%2C40681%2C1732422576481.1732422592463
2024-11-24T04:29:52,466 WARN  [Thread-910 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741841_1023
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:29:52,466 WARN  [Thread-910 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1909738581-172.17.0.2-1732422575571:blk_1073741841_1023 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK], DatanodeInfoWithStorage[127.0.0.1:45685,DS-48952f18-74aa-43bd-ad38-e5a4deb9635a,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]) is bad.
2024-11-24T04:29:52,466 WARN  [Thread-910 {}] hdfs.DataStreamer(1850): Abandoning BP-1909738581-172.17.0.2-1732422575571:blk_1073741841_1023
2024-11-24T04:29:52,467 WARN  [Thread-910 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]
2024-11-24T04:29:52,469 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.1732422576995 after 4007ms
2024-11-24T04:29:52,469 WARN  [Thread-910 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741842_1024
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:36375
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:29:52,469 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_620784384_22 at /127.0.0.1:49750 [Receiving block BP-1909738581-172.17.0.2-1732422575571:blk_1073741842_1024] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/cluster_56272640-4b45-3980-6187-aab4c21513ae/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/cluster_56272640-4b45-3980-6187-aab4c21513ae/data/data6]'}, localName='127.0.0.1:35379', datanodeUuid='1fddb4ec-42cd-40c2-8e6d-fb70aba59f5b', xmitsInProgress=0}:Exception transferring block BP-1909738581-172.17.0.2-1732422575571:blk_1073741842_1024 to mirror 127.0.0.1:36375
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T04:29:52,470 WARN  [Thread-910 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1909738581-172.17.0.2-1732422575571:blk_1073741842_1024 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35379,DS-4844294c-4a9d-4087-a05b-bdb1c1eed91d,DISK], DatanodeInfoWithStorage[127.0.0.1:36375,DS-535553ae-5a1c-418e-828a-fe05e393ffd4,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:36375,DS-535553ae-5a1c-418e-828a-fe05e393ffd4,DISK]) is bad.
2024-11-24T04:29:52,470 WARN  [Thread-910 {}] hdfs.DataStreamer(1850): Abandoning BP-1909738581-172.17.0.2-1732422575571:blk_1073741842_1024
2024-11-24T04:29:52,470 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_620784384_22 at /127.0.0.1:49750 [Receiving block BP-1909738581-172.17.0.2-1732422575571:blk_1073741842_1024] {}] datanode.BlockReceiver(316): Block 1073741842 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-24T04:29:52,470 WARN  [Thread-910 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36375,DS-535553ae-5a1c-418e-828a-fe05e393ffd4,DISK]
2024-11-24T04:29:52,470 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_620784384_22 at /127.0.0.1:49750 [Receiving block BP-1909738581-172.17.0.2-1732422575571:blk_1073741842_1024] {}] datanode.DataXceiver(331): 127.0.0.1:35379:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:49750 dst: /127.0.0.1:35379
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T04:29:52,472 WARN  [Thread-910 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741843_1025
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:37029
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:29:52,472 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_620784384_22 at /127.0.0.1:41830 [Receiving block BP-1909738581-172.17.0.2-1732422575571:blk_1073741843_1025] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/cluster_56272640-4b45-3980-6187-aab4c21513ae/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/cluster_56272640-4b45-3980-6187-aab4c21513ae/data/data10]'}, localName='127.0.0.1:45685', datanodeUuid='a280f544-a74c-41cc-82b9-31a5022433e7', xmitsInProgress=0}:Exception transferring block BP-1909738581-172.17.0.2-1732422575571:blk_1073741843_1025 to mirror 127.0.0.1:37029
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T04:29:52,472 WARN  [Thread-910 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1909738581-172.17.0.2-1732422575571:blk_1073741843_1025 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45685,DS-48952f18-74aa-43bd-ad38-e5a4deb9635a,DISK], DatanodeInfoWithStorage[127.0.0.1:37029,DS-1c26aeaf-2106-4f04-a38e-568cd2360db7,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:37029,DS-1c26aeaf-2106-4f04-a38e-568cd2360db7,DISK]) is bad.
2024-11-24T04:29:52,472 WARN  [Thread-910 {}] hdfs.DataStreamer(1850): Abandoning BP-1909738581-172.17.0.2-1732422575571:blk_1073741843_1025
2024-11-24T04:29:52,472 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_620784384_22 at /127.0.0.1:41830 [Receiving block BP-1909738581-172.17.0.2-1732422575571:blk_1073741843_1025] {}] datanode.BlockReceiver(316): Block 1073741843 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-24T04:29:52,473 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_620784384_22 at /127.0.0.1:41830 [Receiving block BP-1909738581-172.17.0.2-1732422575571:blk_1073741843_1025] {}] datanode.DataXceiver(331): 127.0.0.1:45685:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41830 dst: /127.0.0.1:45685
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T04:29:52,473 WARN  [Thread-910 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37029,DS-1c26aeaf-2106-4f04-a38e-568cd2360db7,DISK]
2024-11-24T04:29:52,476 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-11-24T04:29:52,480 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:52,480 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:52,480 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:52,480 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:52,480 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:52,480 INFO  [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.1732422588447 with entries=3, filesize=3.51 KB; new WAL /user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.1732422592463
2024-11-24T04:29:52,482 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35379 is added to blk_1073741840_1022 (size=3600)
2024-11-24T04:29:52,489 DEBUG [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39949:39949),(127.0.0.1/127.0.0.1:33121:33121)]
2024-11-24T04:29:52,489 DEBUG [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.1732422576995 is not closed yet, will try archiving it next time
2024-11-24T04:29:52,489 DEBUG [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.1732422588447 is not closed yet, will try archiving it next time
2024-11-24T04:29:52,587 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:29:52,883 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.1732422576995 is not closed yet, will try archiving it next time
2024-11-24T04:29:53,613 INFO  [regionserver/fadb2fe0ba32:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:29:54,045 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@5df0d8bb[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:35379, datanodeUuid=1fddb4ec-42cd-40c2-8e6d-fb70aba59f5b, infoPort=39949, infoSecurePort=0, ipcPort=43999, storageInfo=lv=-57;cid=testClusterID;nsid=612618959;c=1732422575571):Failed to transfer BP-1909738581-172.17.0.2-1732422575571:blk_1073741840_1022 to 127.0.0.1:36375 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T04:29:54,480 WARN  [ResponseProcessor for block BP-1909738581-172.17.0.2-1732422575571:blk_1073741844_1026 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1909738581-172.17.0.2-1732422575571:blk_1073741844_1026
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:29:54,480 WARN  [DataStreamer for file /user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.1732422592463 block BP-1909738581-172.17.0.2-1732422575571:blk_1073741844_1026 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1909738581-172.17.0.2-1732422575571:blk_1073741844_1026 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35379,DS-4844294c-4a9d-4087-a05b-bdb1c1eed91d,DISK], DatanodeInfoWithStorage[127.0.0.1:45685,DS-48952f18-74aa-43bd-ad38-e5a4deb9635a,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35379,DS-4844294c-4a9d-4087-a05b-bdb1c1eed91d,DISK]) is bad.
2024-11-24T04:29:54,481 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_620784384_22 at /127.0.0.1:49764 [Receiving block BP-1909738581-172.17.0.2-1732422575571:blk_1073741844_1026] {}] datanode.DataXceiver(331): 127.0.0.1:35379:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:49764 dst: /127.0.0.1:35379
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T04:29:54,481 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_620784384_22 at /127.0.0.1:41844 [Receiving block BP-1909738581-172.17.0.2-1732422575571:blk_1073741844_1026] {}] datanode.DataXceiver(331): 127.0.0.1:45685:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41844 dst: /127.0.0.1:45685
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T04:29:54,482 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3c5438f9{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T04:29:54,483 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2e7a8425{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-24T04:29:54,483 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-24T04:29:54,483 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@51f59516{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-24T04:29:54,483 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@36d0b5ff{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/hadoop.log.dir/,STOPPED}
2024-11-24T04:29:54,484 WARN  [BP-1909738581-172.17.0.2-1732422575571 heartbeating to localhost/127.0.0.1:40187 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-24T04:29:54,484 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-24T04:29:54,485 WARN  [BP-1909738581-172.17.0.2-1732422575571 heartbeating to localhost/127.0.0.1:40187 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1909738581-172.17.0.2-1732422575571 (Datanode Uuid 1fddb4ec-42cd-40c2-8e6d-fb70aba59f5b) service to localhost/127.0.0.1:40187
2024-11-24T04:29:54,485 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-24T04:29:54,485 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/cluster_56272640-4b45-3980-6187-aab4c21513ae/data/data5/current/BP-1909738581-172.17.0.2-1732422575571 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T04:29:54,485 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/cluster_56272640-4b45-3980-6187-aab4c21513ae/data/data6/current/BP-1909738581-172.17.0.2-1732422575571 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T04:29:54,485 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-24T04:29:54,489 INFO  [regionserver/fadb2fe0ba32:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:29:54,489 WARN  [regionserver/fadb2fe0ba32:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:45685,DS-48952f18-74aa-43bd-ad38-e5a4deb9635a,DISK]]
2024-11-24T04:29:54,489 DEBUG [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog fadb2fe0ba32%2C40681%2C1732422576481:(num 1732422592463) roll requested
2024-11-24T04:29:54,490 INFO  [regionserver/fadb2fe0ba32:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor fadb2fe0ba32%2C40681%2C1732422576481.1732422594489
2024-11-24T04:29:54,492 WARN  [Thread-924 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741845_1028
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:29:54,493 WARN  [Thread-924 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1909738581-172.17.0.2-1732422575571:blk_1073741845_1028 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35379,DS-4844294c-4a9d-4087-a05b-bdb1c1eed91d,DISK], DatanodeInfoWithStorage[127.0.0.1:36375,DS-535553ae-5a1c-418e-828a-fe05e393ffd4,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35379,DS-4844294c-4a9d-4087-a05b-bdb1c1eed91d,DISK]) is bad.
2024-11-24T04:29:54,493 WARN  [Thread-924 {}] hdfs.DataStreamer(1850): Abandoning BP-1909738581-172.17.0.2-1732422575571:blk_1073741845_1028
2024-11-24T04:29:54,493 WARN  [Thread-924 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35379,DS-4844294c-4a9d-4087-a05b-bdb1c1eed91d,DISK]
2024-11-24T04:29:54,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40681 {}] regionserver.HRegion(8855): Flush requested on 0ce341802081e9b50470c63603e22c62
2024-11-24T04:29:54,494 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0ce341802081e9b50470c63603e22c62 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-24T04:29:54,505 WARN  [Thread-924 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741846_1029
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:46879
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:29:54,505 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_620784384_22 at /127.0.0.1:41866 [Receiving block BP-1909738581-172.17.0.2-1732422575571:blk_1073741846_1029] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/cluster_56272640-4b45-3980-6187-aab4c21513ae/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/cluster_56272640-4b45-3980-6187-aab4c21513ae/data/data10]'}, localName='127.0.0.1:45685', datanodeUuid='a280f544-a74c-41cc-82b9-31a5022433e7', xmitsInProgress=0}:Exception transferring block BP-1909738581-172.17.0.2-1732422575571:blk_1073741846_1029 to mirror 127.0.0.1:46879
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T04:29:54,505 WARN  [Thread-924 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1909738581-172.17.0.2-1732422575571:blk_1073741846_1029 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45685,DS-48952f18-74aa-43bd-ad38-e5a4deb9635a,DISK], DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]) is bad.
2024-11-24T04:29:54,505 WARN  [Thread-924 {}] hdfs.DataStreamer(1850): Abandoning BP-1909738581-172.17.0.2-1732422575571:blk_1073741846_1029
2024-11-24T04:29:54,505 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_620784384_22 at /127.0.0.1:41866 [Receiving block BP-1909738581-172.17.0.2-1732422575571:blk_1073741846_1029] {}] datanode.BlockReceiver(316): Block 1073741846 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-24T04:29:54,506 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_620784384_22 at /127.0.0.1:41866 [Receiving block BP-1909738581-172.17.0.2-1732422575571:blk_1073741846_1029] {}] datanode.DataXceiver(331): 127.0.0.1:45685:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41866 dst: /127.0.0.1:45685
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T04:29:54,506 WARN  [Thread-924 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]
2024-11-24T04:29:54,507 WARN  [Thread-924 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741847_1030
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:29:54,508 WARN  [Thread-924 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1909738581-172.17.0.2-1732422575571:blk_1073741847_1030 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36375,DS-535553ae-5a1c-418e-828a-fe05e393ffd4,DISK], DatanodeInfoWithStorage[127.0.0.1:37029,DS-1c26aeaf-2106-4f04-a38e-568cd2360db7,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36375,DS-535553ae-5a1c-418e-828a-fe05e393ffd4,DISK]) is bad.
2024-11-24T04:29:54,508 WARN  [Thread-924 {}] hdfs.DataStreamer(1850): Abandoning BP-1909738581-172.17.0.2-1732422575571:blk_1073741847_1030
2024-11-24T04:29:54,508 WARN  [Thread-924 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36375,DS-535553ae-5a1c-418e-828a-fe05e393ffd4,DISK]
2024-11-24T04:29:54,509 WARN  [Thread-924 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741848_1031
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:29:54,510 WARN  [Thread-924 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1909738581-172.17.0.2-1732422575571:blk_1073741848_1031 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37029,DS-1c26aeaf-2106-4f04-a38e-568cd2360db7,DISK], DatanodeInfoWithStorage[127.0.0.1:45685,DS-48952f18-74aa-43bd-ad38-e5a4deb9635a,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37029,DS-1c26aeaf-2106-4f04-a38e-568cd2360db7,DISK]) is bad.
2024-11-24T04:29:54,510 WARN  [Thread-924 {}] hdfs.DataStreamer(1850): Abandoning BP-1909738581-172.17.0.2-1732422575571:blk_1073741848_1031
2024-11-24T04:29:54,510 WARN  [Thread-924 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37029,DS-1c26aeaf-2106-4f04-a38e-568cd2360db7,DISK]
2024-11-24T04:29:54,511 WARN  [IPC Server handler 4 on default port 40187 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-24T04:29:54,511 WARN  [IPC Server handler 4 on default port 40187 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-24T04:29:54,512 WARN  [IPC Server handler 4 on default port 40187 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-24T04:29:54,518 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:54,518 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:54,518 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:54,518 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:54,519 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:54,519 INFO  [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.1732422592463 with entries=12, filesize=12.96 KB; new WAL /user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.1732422594489
2024-11-24T04:29:54,520 DEBUG [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33121:33121)]
2024-11-24T04:29:54,520 DEBUG [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.1732422576995 is not closed yet, will try archiving it next time
2024-11-24T04:29:54,520 DEBUG [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.1732422592463 is not closed yet, will try archiving it next time
2024-11-24T04:29:54,520 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/.tmp/info/b81a18220de8466091d6f09693816a0b is 1080, key is row0002/info:/1732422590472/Put/seqid=0
2024-11-24T04:29:54,521 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45685 is added to blk_1073741844_1027 (size=13274)
2024-11-24T04:29:54,522 WARN  [Thread-926 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741850_1033
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:29:54,522 WARN  [Thread-926 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1909738581-172.17.0.2-1732422575571:blk_1073741850_1033 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35379,DS-4844294c-4a9d-4087-a05b-bdb1c1eed91d,DISK], DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35379,DS-4844294c-4a9d-4087-a05b-bdb1c1eed91d,DISK]) is bad.
2024-11-24T04:29:54,522 WARN  [Thread-926 {}] hdfs.DataStreamer(1850): Abandoning BP-1909738581-172.17.0.2-1732422575571:blk_1073741850_1033
2024-11-24T04:29:54,523 WARN  [Thread-926 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35379,DS-4844294c-4a9d-4087-a05b-bdb1c1eed91d,DISK]
2024-11-24T04:29:54,524 WARN  [Thread-926 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741851_1034
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:29:54,524 WARN  [Thread-926 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1909738581-172.17.0.2-1732422575571:blk_1073741851_1034 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK], DatanodeInfoWithStorage[127.0.0.1:36375,DS-535553ae-5a1c-418e-828a-fe05e393ffd4,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]) is bad.
2024-11-24T04:29:54,524 WARN  [Thread-926 {}] hdfs.DataStreamer(1850): Abandoning BP-1909738581-172.17.0.2-1732422575571:blk_1073741851_1034
2024-11-24T04:29:54,525 WARN  [Thread-926 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]
2024-11-24T04:29:54,526 WARN  [Thread-926 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741852_1035
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:29:54,526 WARN  [Thread-926 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1909738581-172.17.0.2-1732422575571:blk_1073741852_1035 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37029,DS-1c26aeaf-2106-4f04-a38e-568cd2360db7,DISK], DatanodeInfoWithStorage[127.0.0.1:45685,DS-48952f18-74aa-43bd-ad38-e5a4deb9635a,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37029,DS-1c26aeaf-2106-4f04-a38e-568cd2360db7,DISK]) is bad.
2024-11-24T04:29:54,526 WARN  [Thread-926 {}] hdfs.DataStreamer(1850): Abandoning BP-1909738581-172.17.0.2-1732422575571:blk_1073741852_1035
2024-11-24T04:29:54,527 WARN  [Thread-926 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37029,DS-1c26aeaf-2106-4f04-a38e-568cd2360db7,DISK]
2024-11-24T04:29:54,528 WARN  [Thread-926 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741853_1036
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:29:54,528 WARN  [Thread-926 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1909738581-172.17.0.2-1732422575571:blk_1073741853_1036 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36375,DS-535553ae-5a1c-418e-828a-fe05e393ffd4,DISK], DatanodeInfoWithStorage[127.0.0.1:45685,DS-48952f18-74aa-43bd-ad38-e5a4deb9635a,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36375,DS-535553ae-5a1c-418e-828a-fe05e393ffd4,DISK]) is bad.
2024-11-24T04:29:54,528 WARN  [Thread-926 {}] hdfs.DataStreamer(1850): Abandoning BP-1909738581-172.17.0.2-1732422575571:blk_1073741853_1036
2024-11-24T04:29:54,529 WARN  [Thread-926 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36375,DS-535553ae-5a1c-418e-828a-fe05e393ffd4,DISK]
2024-11-24T04:29:54,529 WARN  [IPC Server handler 3 on default port 40187 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-24T04:29:54,529 WARN  [IPC Server handler 3 on default port 40187 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-24T04:29:54,529 WARN  [IPC Server handler 3 on default port 40187 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-24T04:29:54,533 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45685 is added to blk_1073741854_1037 (size=10347)
2024-11-24T04:29:54,533 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/.tmp/info/b81a18220de8466091d6f09693816a0b
2024-11-24T04:29:54,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/.tmp/info/b81a18220de8466091d6f09693816a0b as hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/info/b81a18220de8466091d6f09693816a0b
2024-11-24T04:29:54,546 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/info/b81a18220de8466091d6f09693816a0b, entries=5, sequenceid=11, filesize=10.1 K
2024-11-24T04:29:54,548 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=7.35 KB/7531 for 0ce341802081e9b50470c63603e22c62 in 53ms, sequenceid=11, compaction requested=false
2024-11-24T04:29:54,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0ce341802081e9b50470c63603e22c62:

2024-11-24T04:29:54,587 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:29:54,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40681 {}] regionserver.HRegion(8855): Flush requested on 0ce341802081e9b50470c63603e22c62
2024-11-24T04:29:54,712 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0ce341802081e9b50470c63603e22c62 1/1 column families, dataSize=8.40 KB heapSize=9.25 KB
2024-11-24T04:29:54,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/.tmp/info/33e16ffd3dfd40b7be285078a5953437 is 1080, key is row0007/info:/1732422594495/Put/seqid=0
2024-11-24T04:29:54,719 WARN  [Thread-934 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741855_1038
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:29:54,719 WARN  [Thread-934 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1909738581-172.17.0.2-1732422575571:blk_1073741855_1038 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35379,DS-4844294c-4a9d-4087-a05b-bdb1c1eed91d,DISK], DatanodeInfoWithStorage[127.0.0.1:37029,DS-1c26aeaf-2106-4f04-a38e-568cd2360db7,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35379,DS-4844294c-4a9d-4087-a05b-bdb1c1eed91d,DISK]) is bad.
2024-11-24T04:29:54,719 WARN  [Thread-934 {}] hdfs.DataStreamer(1850): Abandoning BP-1909738581-172.17.0.2-1732422575571:blk_1073741855_1038
2024-11-24T04:29:54,720 WARN  [Thread-934 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35379,DS-4844294c-4a9d-4087-a05b-bdb1c1eed91d,DISK]
2024-11-24T04:29:54,721 WARN  [Thread-934 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741856_1039
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:29:54,721 WARN  [Thread-934 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1909738581-172.17.0.2-1732422575571:blk_1073741856_1039 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK], DatanodeInfoWithStorage[127.0.0.1:37029,DS-1c26aeaf-2106-4f04-a38e-568cd2360db7,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]) is bad.
2024-11-24T04:29:54,721 WARN  [Thread-934 {}] hdfs.DataStreamer(1850): Abandoning BP-1909738581-172.17.0.2-1732422575571:blk_1073741856_1039
2024-11-24T04:29:54,722 WARN  [Thread-934 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]
2024-11-24T04:29:54,723 WARN  [Thread-934 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741857_1040
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:29:54,723 WARN  [Thread-934 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1909738581-172.17.0.2-1732422575571:blk_1073741857_1040 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37029,DS-1c26aeaf-2106-4f04-a38e-568cd2360db7,DISK], DatanodeInfoWithStorage[127.0.0.1:45685,DS-48952f18-74aa-43bd-ad38-e5a4deb9635a,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37029,DS-1c26aeaf-2106-4f04-a38e-568cd2360db7,DISK]) is bad.
2024-11-24T04:29:54,723 WARN  [Thread-934 {}] hdfs.DataStreamer(1850): Abandoning BP-1909738581-172.17.0.2-1732422575571:blk_1073741857_1040
2024-11-24T04:29:54,723 WARN  [Thread-934 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37029,DS-1c26aeaf-2106-4f04-a38e-568cd2360db7,DISK]
2024-11-24T04:29:54,725 WARN  [Thread-934 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741858_1041
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:36375
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:29:54,725 WARN  [Thread-934 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1909738581-172.17.0.2-1732422575571:blk_1073741858_1041 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45685,DS-48952f18-74aa-43bd-ad38-e5a4deb9635a,DISK], DatanodeInfoWithStorage[127.0.0.1:36375,DS-535553ae-5a1c-418e-828a-fe05e393ffd4,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:36375,DS-535553ae-5a1c-418e-828a-fe05e393ffd4,DISK]) is bad.
2024-11-24T04:29:54,725 WARN  [Thread-934 {}] hdfs.DataStreamer(1850): Abandoning BP-1909738581-172.17.0.2-1732422575571:blk_1073741858_1041
2024-11-24T04:29:54,725 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_620784384_22 at /127.0.0.1:41898 [Receiving block BP-1909738581-172.17.0.2-1732422575571:blk_1073741858_1041] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/cluster_56272640-4b45-3980-6187-aab4c21513ae/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/cluster_56272640-4b45-3980-6187-aab4c21513ae/data/data10]'}, localName='127.0.0.1:45685', datanodeUuid='a280f544-a74c-41cc-82b9-31a5022433e7', xmitsInProgress=0}:Exception transferring block BP-1909738581-172.17.0.2-1732422575571:blk_1073741858_1041 to mirror 127.0.0.1:36375
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T04:29:54,726 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_620784384_22 at /127.0.0.1:41898 [Receiving block BP-1909738581-172.17.0.2-1732422575571:blk_1073741858_1041] {}] datanode.BlockReceiver(316): Block 1073741858 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-24T04:29:54,726 WARN  [Thread-934 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36375,DS-535553ae-5a1c-418e-828a-fe05e393ffd4,DISK]
2024-11-24T04:29:54,726 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_620784384_22 at /127.0.0.1:41898 [Receiving block BP-1909738581-172.17.0.2-1732422575571:blk_1073741858_1041] {}] datanode.DataXceiver(331): 127.0.0.1:45685:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41898 dst: /127.0.0.1:45685
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T04:29:54,726 WARN  [IPC Server handler 0 on default port 40187 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-24T04:29:54,726 WARN  [IPC Server handler 0 on default port 40187 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-24T04:29:54,727 WARN  [IPC Server handler 0 on default port 40187 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-24T04:29:54,730 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45685 is added to blk_1073741859_1042 (size=12506)
2024-11-24T04:29:54,922 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.1732422576995 is not closed yet, will try archiving it next time
2024-11-24T04:29:54,922 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.1732422588447 to hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/oldWALs/fadb2fe0ba32%2C40681%2C1732422576481.1732422588447
2024-11-24T04:29:55,131 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.40 KB at sequenceid=22 (bloomFilter=true), to=hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/.tmp/info/33e16ffd3dfd40b7be285078a5953437
2024-11-24T04:29:55,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/.tmp/info/33e16ffd3dfd40b7be285078a5953437 as hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/info/33e16ffd3dfd40b7be285078a5953437
2024-11-24T04:29:55,143 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/info/33e16ffd3dfd40b7be285078a5953437, entries=7, sequenceid=22, filesize=12.2 K
2024-11-24T04:29:55,145 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~8.40 KB/8606, heapSize ~9.23 KB/9456, currentSize=2.10 KB/2150 for 0ce341802081e9b50470c63603e22c62 in 433ms, sequenceid=22, compaction requested=false
2024-11-24T04:29:55,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0ce341802081e9b50470c63603e22c62:

2024-11-24T04:29:55,145 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=22.3 K, sizeToCheck=16.0 K
2024-11-24T04:29:55,145 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-24T04:29:55,145 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/info/33e16ffd3dfd40b7be285078a5953437 because midkey is the same as first or last row
2024-11-24T04:29:55,613 INFO  [regionserver/fadb2fe0ba32:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:29:56,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40681 {}] regionserver.HRegion(8855): Flush requested on 0ce341802081e9b50470c63603e22c62
2024-11-24T04:29:56,132 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0ce341802081e9b50470c63603e22c62 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-11-24T04:29:56,137 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/.tmp/info/d6b5a3734c0e4e0495e0d804f22d7ae4 is 1079, key is tmprow/info:/1732422596130/Put/seqid=0
2024-11-24T04:29:56,140 WARN  [Thread-939 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741860_1043
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:46879
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:29:56,140 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_620784384_22 at /127.0.0.1:41918 [Receiving block BP-1909738581-172.17.0.2-1732422575571:blk_1073741860_1043] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/cluster_56272640-4b45-3980-6187-aab4c21513ae/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/cluster_56272640-4b45-3980-6187-aab4c21513ae/data/data10]'}, localName='127.0.0.1:45685', datanodeUuid='a280f544-a74c-41cc-82b9-31a5022433e7', xmitsInProgress=0}:Exception transferring block BP-1909738581-172.17.0.2-1732422575571:blk_1073741860_1043 to mirror 127.0.0.1:46879
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T04:29:56,140 WARN  [Thread-939 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1909738581-172.17.0.2-1732422575571:blk_1073741860_1043 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45685,DS-48952f18-74aa-43bd-ad38-e5a4deb9635a,DISK], DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]) is bad.
2024-11-24T04:29:56,140 WARN  [Thread-939 {}] hdfs.DataStreamer(1850): Abandoning BP-1909738581-172.17.0.2-1732422575571:blk_1073741860_1043
2024-11-24T04:29:56,140 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_620784384_22 at /127.0.0.1:41918 [Receiving block BP-1909738581-172.17.0.2-1732422575571:blk_1073741860_1043] {}] datanode.BlockReceiver(316): Block 1073741860 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-24T04:29:56,140 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_620784384_22 at /127.0.0.1:41918 [Receiving block BP-1909738581-172.17.0.2-1732422575571:blk_1073741860_1043] {}] datanode.DataXceiver(331): 127.0.0.1:45685:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41918 dst: /127.0.0.1:45685
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T04:29:56,141 WARN  [Thread-939 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]
2024-11-24T04:29:56,142 WARN  [Thread-939 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741861_1044
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:29:56,142 WARN  [Thread-939 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1909738581-172.17.0.2-1732422575571:blk_1073741861_1044 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35379,DS-4844294c-4a9d-4087-a05b-bdb1c1eed91d,DISK], DatanodeInfoWithStorage[127.0.0.1:45685,DS-48952f18-74aa-43bd-ad38-e5a4deb9635a,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35379,DS-4844294c-4a9d-4087-a05b-bdb1c1eed91d,DISK]) is bad.
2024-11-24T04:29:56,142 WARN  [Thread-939 {}] hdfs.DataStreamer(1850): Abandoning BP-1909738581-172.17.0.2-1732422575571:blk_1073741861_1044
2024-11-24T04:29:56,143 WARN  [Thread-939 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35379,DS-4844294c-4a9d-4087-a05b-bdb1c1eed91d,DISK]
2024-11-24T04:29:56,144 WARN  [Thread-939 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741862_1045
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:29:56,144 WARN  [Thread-939 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1909738581-172.17.0.2-1732422575571:blk_1073741862_1045 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37029,DS-1c26aeaf-2106-4f04-a38e-568cd2360db7,DISK], DatanodeInfoWithStorage[127.0.0.1:36375,DS-535553ae-5a1c-418e-828a-fe05e393ffd4,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37029,DS-1c26aeaf-2106-4f04-a38e-568cd2360db7,DISK]) is bad.
2024-11-24T04:29:56,144 WARN  [Thread-939 {}] hdfs.DataStreamer(1850): Abandoning BP-1909738581-172.17.0.2-1732422575571:blk_1073741862_1045
2024-11-24T04:29:56,145 WARN  [Thread-939 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37029,DS-1c26aeaf-2106-4f04-a38e-568cd2360db7,DISK]
2024-11-24T04:29:56,146 WARN  [Thread-939 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741863_1046
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:29:56,146 WARN  [Thread-939 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1909738581-172.17.0.2-1732422575571:blk_1073741863_1046 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36375,DS-535553ae-5a1c-418e-828a-fe05e393ffd4,DISK], DatanodeInfoWithStorage[127.0.0.1:45685,DS-48952f18-74aa-43bd-ad38-e5a4deb9635a,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36375,DS-535553ae-5a1c-418e-828a-fe05e393ffd4,DISK]) is bad.
2024-11-24T04:29:56,146 WARN  [Thread-939 {}] hdfs.DataStreamer(1850): Abandoning BP-1909738581-172.17.0.2-1732422575571:blk_1073741863_1046
2024-11-24T04:29:56,147 WARN  [Thread-939 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36375,DS-535553ae-5a1c-418e-828a-fe05e393ffd4,DISK]
2024-11-24T04:29:56,147 WARN  [IPC Server handler 0 on default port 40187 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-24T04:29:56,147 WARN  [IPC Server handler 0 on default port 40187 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-24T04:29:56,148 WARN  [IPC Server handler 0 on default port 40187 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-24T04:29:56,150 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45685 is added to blk_1073741864_1047 (size=6027)
2024-11-24T04:29:56,520 INFO  [regionserver/fadb2fe0ba32:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:29:56,520 WARN  [regionserver/fadb2fe0ba32:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:45685,DS-48952f18-74aa-43bd-ad38-e5a4deb9635a,DISK]]
2024-11-24T04:29:56,521 DEBUG [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog fadb2fe0ba32%2C40681%2C1732422576481:(num 1732422594489) roll requested
2024-11-24T04:29:56,521 INFO  [regionserver/fadb2fe0ba32:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor fadb2fe0ba32%2C40681%2C1732422576481.1732422596521
2024-11-24T04:29:56,524 WARN  [Thread-944 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741865_1048
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:29:56,524 WARN  [Thread-944 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1909738581-172.17.0.2-1732422575571:blk_1073741865_1048 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35379,DS-4844294c-4a9d-4087-a05b-bdb1c1eed91d,DISK], DatanodeInfoWithStorage[127.0.0.1:36375,DS-535553ae-5a1c-418e-828a-fe05e393ffd4,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35379,DS-4844294c-4a9d-4087-a05b-bdb1c1eed91d,DISK]) is bad.
2024-11-24T04:29:56,524 WARN  [Thread-944 {}] hdfs.DataStreamer(1850): Abandoning BP-1909738581-172.17.0.2-1732422575571:blk_1073741865_1048
2024-11-24T04:29:56,525 WARN  [Thread-944 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35379,DS-4844294c-4a9d-4087-a05b-bdb1c1eed91d,DISK]
2024-11-24T04:29:56,527 WARN  [Thread-944 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741866_1049
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:36375
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:29:56,527 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_620784384_22 at /127.0.0.1:41924 [Receiving block BP-1909738581-172.17.0.2-1732422575571:blk_1073741866_1049] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/cluster_56272640-4b45-3980-6187-aab4c21513ae/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/cluster_56272640-4b45-3980-6187-aab4c21513ae/data/data10]'}, localName='127.0.0.1:45685', datanodeUuid='a280f544-a74c-41cc-82b9-31a5022433e7', xmitsInProgress=0}:Exception transferring block BP-1909738581-172.17.0.2-1732422575571:blk_1073741866_1049 to mirror 127.0.0.1:36375
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T04:29:56,527 WARN  [Thread-944 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1909738581-172.17.0.2-1732422575571:blk_1073741866_1049 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45685,DS-48952f18-74aa-43bd-ad38-e5a4deb9635a,DISK], DatanodeInfoWithStorage[127.0.0.1:36375,DS-535553ae-5a1c-418e-828a-fe05e393ffd4,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:36375,DS-535553ae-5a1c-418e-828a-fe05e393ffd4,DISK]) is bad.
2024-11-24T04:29:56,527 WARN  [Thread-944 {}] hdfs.DataStreamer(1850): Abandoning BP-1909738581-172.17.0.2-1732422575571:blk_1073741866_1049
2024-11-24T04:29:56,527 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_620784384_22 at /127.0.0.1:41924 [Receiving block BP-1909738581-172.17.0.2-1732422575571:blk_1073741866_1049] {}] datanode.BlockReceiver(316): Block 1073741866 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-24T04:29:56,527 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_620784384_22 at /127.0.0.1:41924 [Receiving block BP-1909738581-172.17.0.2-1732422575571:blk_1073741866_1049] {}] datanode.DataXceiver(331): 127.0.0.1:45685:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41924 dst: /127.0.0.1:45685
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T04:29:56,527 WARN  [Thread-944 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36375,DS-535553ae-5a1c-418e-828a-fe05e393ffd4,DISK]
2024-11-24T04:29:56,528 WARN  [Thread-944 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741867_1050
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:29:56,528 WARN  [Thread-944 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1909738581-172.17.0.2-1732422575571:blk_1073741867_1050 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37029,DS-1c26aeaf-2106-4f04-a38e-568cd2360db7,DISK], DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37029,DS-1c26aeaf-2106-4f04-a38e-568cd2360db7,DISK]) is bad.
2024-11-24T04:29:56,528 WARN  [Thread-944 {}] hdfs.DataStreamer(1850): Abandoning BP-1909738581-172.17.0.2-1732422575571:blk_1073741867_1050
2024-11-24T04:29:56,529 WARN  [Thread-944 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37029,DS-1c26aeaf-2106-4f04-a38e-568cd2360db7,DISK]
2024-11-24T04:29:56,531 WARN  [Thread-944 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741868_1051
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:46879
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:29:56,531 WARN  [Thread-944 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1909738581-172.17.0.2-1732422575571:blk_1073741868_1051 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45685,DS-48952f18-74aa-43bd-ad38-e5a4deb9635a,DISK], DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]) is bad.
2024-11-24T04:29:56,531 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_620784384_22 at /127.0.0.1:41928 [Receiving block BP-1909738581-172.17.0.2-1732422575571:blk_1073741868_1051] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/cluster_56272640-4b45-3980-6187-aab4c21513ae/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/cluster_56272640-4b45-3980-6187-aab4c21513ae/data/data10]'}, localName='127.0.0.1:45685', datanodeUuid='a280f544-a74c-41cc-82b9-31a5022433e7', xmitsInProgress=0}:Exception transferring block BP-1909738581-172.17.0.2-1732422575571:blk_1073741868_1051 to mirror 127.0.0.1:46879
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T04:29:56,531 WARN  [Thread-944 {}] hdfs.DataStreamer(1850): Abandoning BP-1909738581-172.17.0.2-1732422575571:blk_1073741868_1051
2024-11-24T04:29:56,531 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_620784384_22 at /127.0.0.1:41928 [Receiving block BP-1909738581-172.17.0.2-1732422575571:blk_1073741868_1051] {}] datanode.BlockReceiver(316): Block 1073741868 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-24T04:29:56,531 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_620784384_22 at /127.0.0.1:41928 [Receiving block BP-1909738581-172.17.0.2-1732422575571:blk_1073741868_1051] {}] datanode.DataXceiver(331): 127.0.0.1:45685:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41928 dst: /127.0.0.1:45685
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T04:29:56,531 WARN  [Thread-944 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]
2024-11-24T04:29:56,532 WARN  [IPC Server handler 4 on default port 40187 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-24T04:29:56,532 WARN  [IPC Server handler 4 on default port 40187 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-24T04:29:56,532 WARN  [IPC Server handler 4 on default port 40187 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-24T04:29:56,534 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:56,534 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:56,535 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:56,535 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:56,535 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:29:56,535 INFO  [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.1732422594489 with entries=13, filesize=11.68 KB; new WAL /user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.1732422596521
2024-11-24T04:29:56,536 DEBUG [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33121:33121)]
2024-11-24T04:29:56,536 DEBUG [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.1732422576995 is not closed yet, will try archiving it next time
2024-11-24T04:29:56,536 DEBUG [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.1732422594489 is not closed yet, will try archiving it next time
2024-11-24T04:29:56,537 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.1732422592463 to hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/oldWALs/fadb2fe0ba32%2C40681%2C1732422576481.1732422592463
2024-11-24T04:29:56,537 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45685 is added to blk_1073741849_1032 (size=11965)
2024-11-24T04:29:56,551 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/.tmp/info/d6b5a3734c0e4e0495e0d804f22d7ae4
2024-11-24T04:29:56,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/.tmp/info/d6b5a3734c0e4e0495e0d804f22d7ae4 as hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/info/d6b5a3734c0e4e0495e0d804f22d7ae4
2024-11-24T04:29:56,564 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/info/d6b5a3734c0e4e0495e0d804f22d7ae4, entries=1, sequenceid=32, filesize=5.9 K
2024-11-24T04:29:56,565 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for 0ce341802081e9b50470c63603e22c62 in 433ms, sequenceid=32, compaction requested=true
2024-11-24T04:29:56,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0ce341802081e9b50470c63603e22c62:

2024-11-24T04:29:56,565 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.2 K, sizeToCheck=16.0 K
2024-11-24T04:29:56,565 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-24T04:29:56,565 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/info/33e16ffd3dfd40b7be285078a5953437 because midkey is the same as first or last row
2024-11-24T04:29:56,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ce341802081e9b50470c63603e22c62:info, priority=-2147483648, current under compaction store size is 1
2024-11-24T04:29:56,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-24T04:29:56,566 DEBUG [RS:0;fadb2fe0ba32:40681-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-24T04:29:56,567 DEBUG [RS:0;fadb2fe0ba32:40681-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 28880 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-24T04:29:56,567 DEBUG [RS:0;fadb2fe0ba32:40681-shortCompactions-0 {}] regionserver.HStore(1541): 0ce341802081e9b50470c63603e22c62/info is initiating minor compaction (all files)
2024-11-24T04:29:56,567 INFO  [RS:0;fadb2fe0ba32:40681-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ce341802081e9b50470c63603e22c62/info in TestLogRolling-testLogRollOnDatanodeDeath,,1732422577648.0ce341802081e9b50470c63603e22c62.
2024-11-24T04:29:56,568 INFO  [RS:0;fadb2fe0ba32:40681-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/info/b81a18220de8466091d6f09693816a0b, hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/info/33e16ffd3dfd40b7be285078a5953437, hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/info/d6b5a3734c0e4e0495e0d804f22d7ae4] into tmpdir=hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/.tmp, totalSize=28.2 K
2024-11-24T04:29:56,568 DEBUG [RS:0;fadb2fe0ba32:40681-shortCompactions-0 {}] compactions.Compactor(225): Compacting b81a18220de8466091d6f09693816a0b, keycount=5, bloomtype=ROW, size=10.1 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732422590472
2024-11-24T04:29:56,568 DEBUG [RS:0;fadb2fe0ba32:40681-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33e16ffd3dfd40b7be285078a5953437, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=22, earliestPutTs=1732422594495
2024-11-24T04:29:56,569 DEBUG [RS:0;fadb2fe0ba32:40681-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6b5a3734c0e4e0495e0d804f22d7ae4, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=32, earliestPutTs=1732422596130
2024-11-24T04:29:56,583 INFO  [RS:0;fadb2fe0ba32:40681-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ce341802081e9b50470c63603e22c62#info#compaction#21 average throughput is 6.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-24T04:29:56,583 DEBUG [RS:0;fadb2fe0ba32:40681-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/.tmp/info/455b976c3f7d429793f7d5e97510eb27 is 1080, key is row0002/info:/1732422590472/Put/seqid=0
2024-11-24T04:29:56,585 WARN  [Thread-952 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741870_1053
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:46879
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:29:56,585 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_620784384_22 at /127.0.0.1:41962 [Receiving block BP-1909738581-172.17.0.2-1732422575571:blk_1073741870_1053] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/cluster_56272640-4b45-3980-6187-aab4c21513ae/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/cluster_56272640-4b45-3980-6187-aab4c21513ae/data/data10]'}, localName='127.0.0.1:45685', datanodeUuid='a280f544-a74c-41cc-82b9-31a5022433e7', xmitsInProgress=0}:Exception transferring block BP-1909738581-172.17.0.2-1732422575571:blk_1073741870_1053 to mirror 127.0.0.1:46879
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T04:29:56,586 WARN  [Thread-952 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1909738581-172.17.0.2-1732422575571:blk_1073741870_1053 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45685,DS-48952f18-74aa-43bd-ad38-e5a4deb9635a,DISK], DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]) is bad.
2024-11-24T04:29:56,586 WARN  [Thread-952 {}] hdfs.DataStreamer(1850): Abandoning BP-1909738581-172.17.0.2-1732422575571:blk_1073741870_1053
2024-11-24T04:29:56,586 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_620784384_22 at /127.0.0.1:41962 [Receiving block BP-1909738581-172.17.0.2-1732422575571:blk_1073741870_1053] {}] datanode.BlockReceiver(316): Block 1073741870 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-24T04:29:56,586 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_620784384_22 at /127.0.0.1:41962 [Receiving block BP-1909738581-172.17.0.2-1732422575571:blk_1073741870_1053] {}] datanode.DataXceiver(331): 127.0.0.1:45685:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41962 dst: /127.0.0.1:45685
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T04:29:56,586 WARN  [Thread-952 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]
2024-11-24T04:29:56,588 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:29:56,588 WARN  [Thread-952 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741871_1054
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:36375
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:29:56,588 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_620784384_22 at /127.0.0.1:41974 [Receiving block BP-1909738581-172.17.0.2-1732422575571:blk_1073741871_1054] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/cluster_56272640-4b45-3980-6187-aab4c21513ae/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/cluster_56272640-4b45-3980-6187-aab4c21513ae/data/data10]'}, localName='127.0.0.1:45685', datanodeUuid='a280f544-a74c-41cc-82b9-31a5022433e7', xmitsInProgress=0}:Exception transferring block BP-1909738581-172.17.0.2-1732422575571:blk_1073741871_1054 to mirror 127.0.0.1:36375
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T04:29:56,588 WARN  [Thread-952 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1909738581-172.17.0.2-1732422575571:blk_1073741871_1054 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45685,DS-48952f18-74aa-43bd-ad38-e5a4deb9635a,DISK], DatanodeInfoWithStorage[127.0.0.1:36375,DS-535553ae-5a1c-418e-828a-fe05e393ffd4,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:36375,DS-535553ae-5a1c-418e-828a-fe05e393ffd4,DISK]) is bad.
2024-11-24T04:29:56,588 WARN  [Thread-952 {}] hdfs.DataStreamer(1850): Abandoning BP-1909738581-172.17.0.2-1732422575571:blk_1073741871_1054
2024-11-24T04:29:56,588 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_620784384_22 at /127.0.0.1:41974 [Receiving block BP-1909738581-172.17.0.2-1732422575571:blk_1073741871_1054] {}] datanode.BlockReceiver(316): Block 1073741871 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-24T04:29:56,588 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_620784384_22 at /127.0.0.1:41974 [Receiving block BP-1909738581-172.17.0.2-1732422575571:blk_1073741871_1054] {}] datanode.DataXceiver(331): 127.0.0.1:45685:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41974 dst: /127.0.0.1:45685
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T04:29:56,589 WARN  [Thread-952 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36375,DS-535553ae-5a1c-418e-828a-fe05e393ffd4,DISK]
2024-11-24T04:29:56,590 WARN  [Thread-952 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741872_1055
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:29:56,590 WARN  [Thread-952 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1909738581-172.17.0.2-1732422575571:blk_1073741872_1055 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37029,DS-1c26aeaf-2106-4f04-a38e-568cd2360db7,DISK], DatanodeInfoWithStorage[127.0.0.1:45685,DS-48952f18-74aa-43bd-ad38-e5a4deb9635a,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37029,DS-1c26aeaf-2106-4f04-a38e-568cd2360db7,DISK]) is bad.
2024-11-24T04:29:56,590 WARN  [Thread-952 {}] hdfs.DataStreamer(1850): Abandoning BP-1909738581-172.17.0.2-1732422575571:blk_1073741872_1055
2024-11-24T04:29:56,590 WARN  [Thread-952 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37029,DS-1c26aeaf-2106-4f04-a38e-568cd2360db7,DISK]
2024-11-24T04:29:56,591 WARN  [Thread-952 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741873_1056
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:29:56,592 WARN  [Thread-952 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1909738581-172.17.0.2-1732422575571:blk_1073741873_1056 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35379,DS-4844294c-4a9d-4087-a05b-bdb1c1eed91d,DISK], DatanodeInfoWithStorage[127.0.0.1:45685,DS-48952f18-74aa-43bd-ad38-e5a4deb9635a,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35379,DS-4844294c-4a9d-4087-a05b-bdb1c1eed91d,DISK]) is bad.
2024-11-24T04:29:56,592 WARN  [Thread-952 {}] hdfs.DataStreamer(1850): Abandoning BP-1909738581-172.17.0.2-1732422575571:blk_1073741873_1056
2024-11-24T04:29:56,592 WARN  [Thread-952 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35379,DS-4844294c-4a9d-4087-a05b-bdb1c1eed91d,DISK]
2024-11-24T04:29:56,593 WARN  [IPC Server handler 2 on default port 40187 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-24T04:29:56,593 WARN  [IPC Server handler 2 on default port 40187 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-24T04:29:56,593 WARN  [IPC Server handler 2 on default port 40187 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-24T04:29:56,596 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45685 is added to blk_1073741874_1057 (size=17994)
2024-11-24T04:29:56,938 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.1732422576995 is not closed yet, will try archiving it next time
2024-11-24T04:29:57,005 DEBUG [RS:0;fadb2fe0ba32:40681-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/.tmp/info/455b976c3f7d429793f7d5e97510eb27 as hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/info/455b976c3f7d429793f7d5e97510eb27
2024-11-24T04:29:57,014 INFO  [RS:0;fadb2fe0ba32:40681-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0ce341802081e9b50470c63603e22c62/info of 0ce341802081e9b50470c63603e22c62 into 455b976c3f7d429793f7d5e97510eb27(size=17.6 K), total size for store is 17.6 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-24T04:29:57,014 DEBUG [RS:0;fadb2fe0ba32:40681-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ce341802081e9b50470c63603e22c62:

2024-11-24T04:29:57,014 INFO  [RS:0;fadb2fe0ba32:40681-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1732422577648.0ce341802081e9b50470c63603e22c62., storeName=0ce341802081e9b50470c63603e22c62/info, priority=13, startTime=1732422596565; duration=0sec
2024-11-24T04:29:57,014 DEBUG [RS:0;fadb2fe0ba32:40681-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K
2024-11-24T04:29:57,014 DEBUG [RS:0;fadb2fe0ba32:40681-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-24T04:29:57,014 DEBUG [RS:0;fadb2fe0ba32:40681-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/info/455b976c3f7d429793f7d5e97510eb27 because midkey is the same as first or last row
2024-11-24T04:29:57,014 DEBUG [RS:0;fadb2fe0ba32:40681-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K
2024-11-24T04:29:57,014 DEBUG [RS:0;fadb2fe0ba32:40681-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-24T04:29:57,014 DEBUG [RS:0;fadb2fe0ba32:40681-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/info/455b976c3f7d429793f7d5e97510eb27 because midkey is the same as first or last row
2024-11-24T04:29:57,014 DEBUG [RS:0;fadb2fe0ba32:40681-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K
2024-11-24T04:29:57,014 DEBUG [RS:0;fadb2fe0ba32:40681-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-24T04:29:57,014 DEBUG [RS:0;fadb2fe0ba32:40681-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/info/455b976c3f7d429793f7d5e97510eb27 because midkey is the same as first or last row
2024-11-24T04:29:57,014 DEBUG [RS:0;fadb2fe0ba32:40681-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-24T04:29:57,015 DEBUG [RS:0;fadb2fe0ba32:40681-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ce341802081e9b50470c63603e22c62:info
2024-11-24T04:29:57,422 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@4e6fd580[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:45685, datanodeUuid=a280f544-a74c-41cc-82b9-31a5022433e7, infoPort=33121, infoSecurePort=0, ipcPort=39475, storageInfo=lv=-57;cid=testClusterID;nsid=612618959;c=1732422575571):Failed to transfer BP-1909738581-172.17.0.2-1732422575571:blk_1073741844_1027 to 127.0.0.1:37029 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T04:29:57,422 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@4a504895[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:45685, datanodeUuid=a280f544-a74c-41cc-82b9-31a5022433e7, infoPort=33121, infoSecurePort=0, ipcPort=39475, storageInfo=lv=-57;cid=testClusterID;nsid=612618959;c=1732422575571):Failed to transfer BP-1909738581-172.17.0.2-1732422575571:blk_1073741854_1037 to 127.0.0.1:35379 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T04:29:57,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40681 {}] regionserver.HRegion(8855): Flush requested on 0ce341802081e9b50470c63603e22c62
2024-11-24T04:29:57,552 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0ce341802081e9b50470c63603e22c62 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-11-24T04:29:57,557 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/.tmp/info/c98d097882b5455483624014a7841a3f is 1079, key is tmprow/info:/1732422597550/Put/seqid=0
2024-11-24T04:29:57,559 WARN  [Thread-961 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741875_1058
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:29:57,559 WARN  [Thread-961 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1909738581-172.17.0.2-1732422575571:blk_1073741875_1058 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36375,DS-535553ae-5a1c-418e-828a-fe05e393ffd4,DISK], DatanodeInfoWithStorage[127.0.0.1:35379,DS-4844294c-4a9d-4087-a05b-bdb1c1eed91d,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36375,DS-535553ae-5a1c-418e-828a-fe05e393ffd4,DISK]) is bad.
2024-11-24T04:29:57,559 WARN  [Thread-961 {}] hdfs.DataStreamer(1850): Abandoning BP-1909738581-172.17.0.2-1732422575571:blk_1073741875_1058
2024-11-24T04:29:57,559 WARN  [Thread-961 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36375,DS-535553ae-5a1c-418e-828a-fe05e393ffd4,DISK]
2024-11-24T04:29:57,561 WARN  [Thread-961 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741876_1059
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:29:57,561 WARN  [Thread-961 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1909738581-172.17.0.2-1732422575571:blk_1073741876_1059 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK], DatanodeInfoWithStorage[127.0.0.1:35379,DS-4844294c-4a9d-4087-a05b-bdb1c1eed91d,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]) is bad.
2024-11-24T04:29:57,561 WARN  [Thread-961 {}] hdfs.DataStreamer(1850): Abandoning BP-1909738581-172.17.0.2-1732422575571:blk_1073741876_1059
2024-11-24T04:29:57,561 WARN  [Thread-961 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]
2024-11-24T04:29:57,562 WARN  [Thread-961 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741877_1060
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:29:57,563 WARN  [Thread-961 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1909738581-172.17.0.2-1732422575571:blk_1073741877_1060 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35379,DS-4844294c-4a9d-4087-a05b-bdb1c1eed91d,DISK], DatanodeInfoWithStorage[127.0.0.1:37029,DS-1c26aeaf-2106-4f04-a38e-568cd2360db7,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35379,DS-4844294c-4a9d-4087-a05b-bdb1c1eed91d,DISK]) is bad.
2024-11-24T04:29:57,563 WARN  [Thread-961 {}] hdfs.DataStreamer(1850): Abandoning BP-1909738581-172.17.0.2-1732422575571:blk_1073741877_1060
2024-11-24T04:29:57,563 WARN  [Thread-961 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35379,DS-4844294c-4a9d-4087-a05b-bdb1c1eed91d,DISK]
2024-11-24T04:29:57,566 WARN  [Thread-961 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741878_1061
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:37029
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:29:57,566 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_620784384_22 at /127.0.0.1:41984 [Receiving block BP-1909738581-172.17.0.2-1732422575571:blk_1073741878_1061] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/cluster_56272640-4b45-3980-6187-aab4c21513ae/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/cluster_56272640-4b45-3980-6187-aab4c21513ae/data/data10]'}, localName='127.0.0.1:45685', datanodeUuid='a280f544-a74c-41cc-82b9-31a5022433e7', xmitsInProgress=0}:Exception transferring block BP-1909738581-172.17.0.2-1732422575571:blk_1073741878_1061 to mirror 127.0.0.1:37029
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T04:29:57,566 WARN  [Thread-961 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1909738581-172.17.0.2-1732422575571:blk_1073741878_1061 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45685,DS-48952f18-74aa-43bd-ad38-e5a4deb9635a,DISK], DatanodeInfoWithStorage[127.0.0.1:37029,DS-1c26aeaf-2106-4f04-a38e-568cd2360db7,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:37029,DS-1c26aeaf-2106-4f04-a38e-568cd2360db7,DISK]) is bad.
2024-11-24T04:29:57,566 WARN  [Thread-961 {}] hdfs.DataStreamer(1850): Abandoning BP-1909738581-172.17.0.2-1732422575571:blk_1073741878_1061
2024-11-24T04:29:57,566 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_620784384_22 at /127.0.0.1:41984 [Receiving block BP-1909738581-172.17.0.2-1732422575571:blk_1073741878_1061] {}] datanode.BlockReceiver(316): Block 1073741878 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-24T04:29:57,566 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_620784384_22 at /127.0.0.1:41984 [Receiving block BP-1909738581-172.17.0.2-1732422575571:blk_1073741878_1061] {}] datanode.DataXceiver(331): 127.0.0.1:45685:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41984 dst: /127.0.0.1:45685
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T04:29:57,567 WARN  [Thread-961 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37029,DS-1c26aeaf-2106-4f04-a38e-568cd2360db7,DISK]
2024-11-24T04:29:57,567 WARN  [IPC Server handler 3 on default port 40187 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-24T04:29:57,567 WARN  [IPC Server handler 3 on default port 40187 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-24T04:29:57,567 WARN  [IPC Server handler 3 on default port 40187 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-24T04:29:57,570 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45685 is added to blk_1073741879_1062 (size=6027)
2024-11-24T04:29:57,613 INFO  [regionserver/fadb2fe0ba32:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:29:57,972 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=43 (bloomFilter=true), to=hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/.tmp/info/c98d097882b5455483624014a7841a3f
2024-11-24T04:29:57,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/.tmp/info/c98d097882b5455483624014a7841a3f as hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/info/c98d097882b5455483624014a7841a3f
2024-11-24T04:29:57,984 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/info/c98d097882b5455483624014a7841a3f, entries=1, sequenceid=43, filesize=5.9 K
2024-11-24T04:29:57,985 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for 0ce341802081e9b50470c63603e22c62 in 434ms, sequenceid=43, compaction requested=false
2024-11-24T04:29:57,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0ce341802081e9b50470c63603e22c62:

2024-11-24T04:29:57,985 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=23.5 K, sizeToCheck=16.0 K
2024-11-24T04:29:57,985 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-24T04:29:57,985 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/info/455b976c3f7d429793f7d5e97510eb27 because midkey is the same as first or last row
2024-11-24T04:29:58,422 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@4a504895[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:45685, datanodeUuid=a280f544-a74c-41cc-82b9-31a5022433e7, infoPort=33121, infoSecurePort=0, ipcPort=39475, storageInfo=lv=-57;cid=testClusterID;nsid=612618959;c=1732422575571):Failed to transfer BP-1909738581-172.17.0.2-1732422575571:blk_1073741859_1042 to 127.0.0.1:46879 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T04:29:58,422 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@4e6fd580[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:45685, datanodeUuid=a280f544-a74c-41cc-82b9-31a5022433e7, infoPort=33121, infoSecurePort=0, ipcPort=39475, storageInfo=lv=-57;cid=testClusterID;nsid=612618959;c=1732422575571):Failed to transfer BP-1909738581-172.17.0.2-1732422575571:blk_1073741864_1047 to 127.0.0.1:37029 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T04:29:58,536 INFO  [regionserver/fadb2fe0ba32:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:29:58,536 WARN  [regionserver/fadb2fe0ba32:0.logRoller {}] wal.FSHLog(539): Too many consecutive RollWriter requests, it's a sign of the total number of live datanodes is lower than the tolerable replicas.
2024-11-24T04:29:58,573 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T04:29:58,577 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-24T04:29:58,578 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-24T04:29:58,578 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-24T04:29:58,578 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-24T04:29:58,578 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@16a2580d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/hadoop.log.dir/,AVAILABLE}
2024-11-24T04:29:58,579 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@23df32cf{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-24T04:29:58,588 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:29:58,712 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@c1d8e25{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/java.io.tmpdir/jetty-localhost-32791-hadoop-hdfs-3_4_1-tests_jar-_-any-1450308825894944830/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T04:29:58,712 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5af75f98{HTTP/1.1, (http/1.1)}{localhost:32791}
2024-11-24T04:29:58,712 INFO  [Time-limited test {}] server.Server(415): Started @127774ms
2024-11-24T04:29:58,714 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-24T04:29:58,866 WARN  [Thread-980 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-24T04:29:58,876 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xed91215d5dbc9bb with lease ID 0xbac364592180ee89: from storage DS-1c26aeaf-2106-4f04-a38e-568cd2360db7 node DatanodeRegistration(127.0.0.1:39003, datanodeUuid=e3eba06e-e494-45bd-a171-1b16a8edcca7, infoPort=45573, infoSecurePort=0, ipcPort=46113, storageInfo=lv=-57;cid=testClusterID;nsid=612618959;c=1732422575571), blocks: 7, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-24T04:29:58,877 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xed91215d5dbc9bb with lease ID 0xbac364592180ee89: from storage DS-0643a443-b5e3-4fc7-8810-801c75a2afdd node DatanodeRegistration(127.0.0.1:39003, datanodeUuid=e3eba06e-e494-45bd-a171-1b16a8edcca7, infoPort=45573, infoSecurePort=0, ipcPort=46113, storageInfo=lv=-57;cid=testClusterID;nsid=612618959;c=1732422575571), blocks: 6, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-24T04:29:59,614 INFO  [regionserver/fadb2fe0ba32:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:30:00,422 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@4a504895[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:45685, datanodeUuid=a280f544-a74c-41cc-82b9-31a5022433e7, infoPort=33121, infoSecurePort=0, ipcPort=39475, storageInfo=lv=-57;cid=testClusterID;nsid=612618959;c=1732422575571):Failed to transfer BP-1909738581-172.17.0.2-1732422575571:blk_1073741874_1057 to 127.0.0.1:36375 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T04:30:00,426 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39003 is added to blk_1073741849_1032 (size=11965)
2024-11-24T04:30:00,537 INFO  [regionserver/fadb2fe0ba32:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:30:00,588 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:30:01,423 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39003 is added to blk_1073741879_1062 (size=6027)
2024-11-24T04:30:01,614 INFO  [regionserver/fadb2fe0ba32:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:30:02,537 INFO  [regionserver/fadb2fe0ba32:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:30:02,589 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:30:03,615 INFO  [regionserver/fadb2fe0ba32:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:30:04,538 INFO  [regionserver/fadb2fe0ba32:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:30:04,589 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:30:05,615 INFO  [regionserver/fadb2fe0ba32:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:30:06,397 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-24T04:30:06,538 INFO  [regionserver/fadb2fe0ba32:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:30:06,590 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:30:06,750 ERROR [FSHLog-0-hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/MasterData-prefix:fadb2fe0ba32,35377,1732422576418 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:30:06,750 WARN  [FSHLog-0-hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/MasterData-prefix:fadb2fe0ba32,35377,1732422576418 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:30:06,750 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL FSHLog fadb2fe0ba32%2C35377%2C1732422576418:(num 1732422576614) roll requested
2024-11-24T04:30:06,750 INFO  [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor fadb2fe0ba32%2C35377%2C1732422576418.1732422606750
2024-11-24T04:30:06,754 WARN  [Thread-1002 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741880_1063
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:35379
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:30:06,754 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-271356777_22 at /127.0.0.1:46584 [Receiving block BP-1909738581-172.17.0.2-1732422575571:blk_1073741880_1063] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/cluster_56272640-4b45-3980-6187-aab4c21513ae/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/cluster_56272640-4b45-3980-6187-aab4c21513ae/data/data10]'}, localName='127.0.0.1:45685', datanodeUuid='a280f544-a74c-41cc-82b9-31a5022433e7', xmitsInProgress=0}:Exception transferring block BP-1909738581-172.17.0.2-1732422575571:blk_1073741880_1063 to mirror 127.0.0.1:35379
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T04:30:06,754 WARN  [Thread-1002 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1909738581-172.17.0.2-1732422575571:blk_1073741880_1063 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45685,DS-48952f18-74aa-43bd-ad38-e5a4deb9635a,DISK], DatanodeInfoWithStorage[127.0.0.1:35379,DS-4844294c-4a9d-4087-a05b-bdb1c1eed91d,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:35379,DS-4844294c-4a9d-4087-a05b-bdb1c1eed91d,DISK]) is bad.
2024-11-24T04:30:06,754 WARN  [Thread-1002 {}] hdfs.DataStreamer(1850): Abandoning BP-1909738581-172.17.0.2-1732422575571:blk_1073741880_1063
2024-11-24T04:30:06,754 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-271356777_22 at /127.0.0.1:46584 [Receiving block BP-1909738581-172.17.0.2-1732422575571:blk_1073741880_1063] {}] datanode.BlockReceiver(316): Block 1073741880 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-24T04:30:06,754 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-271356777_22 at /127.0.0.1:46584 [Receiving block BP-1909738581-172.17.0.2-1732422575571:blk_1073741880_1063] {}] datanode.DataXceiver(331): 127.0.0.1:45685:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:46584 dst: /127.0.0.1:45685
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T04:30:06,755 WARN  [Thread-1002 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35379,DS-4844294c-4a9d-4087-a05b-bdb1c1eed91d,DISK]
2024-11-24T04:30:06,759 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:06,759 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:06,759 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:06,759 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:06,759 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:06,759 INFO  [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/MasterData/WALs/fadb2fe0ba32,35377,1732422576418/fadb2fe0ba32%2C35377%2C1732422576418.1732422576614 with entries=54, filesize=26.68 KB; new WAL /user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/MasterData/WALs/fadb2fe0ba32,35377,1732422576418/fadb2fe0ba32%2C35377%2C1732422576418.1732422606750
2024-11-24T04:30:06,760 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:30:06,760 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:30:06,760 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/MasterData/WALs/fadb2fe0ba32,35377,1732422576418/fadb2fe0ba32%2C35377%2C1732422576418.1732422576614
2024-11-24T04:30:06,760 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45573:45573),(127.0.0.1/127.0.0.1:33121:33121)]
2024-11-24T04:30:06,760 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/MasterData/WALs/fadb2fe0ba32,35377,1732422576418/fadb2fe0ba32%2C35377%2C1732422576418.1732422576614 is not closed yet, will try archiving it next time
2024-11-24T04:30:06,760 WARN  [IPC Server handler 1 on default port 40187 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/MasterData/WALs/fadb2fe0ba32,35377,1732422576418/fadb2fe0ba32%2C35377%2C1732422576418.1732422576614 has not been closed. Lease recovery is in progress. RecoveryId = 1065 for block blk_1073741830_1006
2024-11-24T04:30:06,761 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/MasterData/WALs/fadb2fe0ba32,35377,1732422576418/fadb2fe0ba32%2C35377%2C1732422576418.1732422576614 after 1ms
2024-11-24T04:30:07,615 INFO  [regionserver/fadb2fe0ba32:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:30:08,538 INFO  [regionserver/fadb2fe0ba32:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:30:08,896 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@6dabd117 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1909738581-172.17.0.2-1732422575571:blk_1073741833_1009, datanode=DatanodeInfoWithStorage[127.0.0.1:46879,null,null])
java.net.ConnectException: Call From fadb2fe0ba32/172.17.0.2 to localhost:46669 failed on connection exception: java.net.ConnectException: Connection refused; For more details see:  http://wiki.apache.org/hadoop/ConnectionRefused
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-11-24T04:30:08,899 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39003 is added to blk_1073741833_1021 (size=455)
2024-11-24T04:30:09,486 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.1732422576995 to hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/oldWALs/fadb2fe0ba32%2C40681%2C1732422576481.1732422576995
2024-11-24T04:30:09,487 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.1732422594489 to hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/oldWALs/fadb2fe0ba32%2C40681%2C1732422576481.1732422594489
2024-11-24T04:30:09,616 INFO  [regionserver/fadb2fe0ba32:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:30:10,539 INFO  [regionserver/fadb2fe0ba32:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:30:10,762 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/MasterData/WALs/fadb2fe0ba32,35377,1732422576418/fadb2fe0ba32%2C35377%2C1732422576418.1732422576614 after 4002ms
2024-11-24T04:30:11,616 INFO  [regionserver/fadb2fe0ba32:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:30:11,870 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@5c8c7869[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:39003, datanodeUuid=e3eba06e-e494-45bd-a171-1b16a8edcca7, infoPort=45573, infoSecurePort=0, ipcPort=46113, storageInfo=lv=-57;cid=testClusterID;nsid=612618959;c=1732422575571):Failed to transfer BP-1909738581-172.17.0.2-1732422575571:blk_1073741833_1021 to 127.0.0.1:35379 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T04:30:11,872 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45685 is added to blk_1073741835_1011 (size=393)
2024-11-24T04:30:12,539 INFO  [regionserver/fadb2fe0ba32:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:30:12,870 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@5c8c7869[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:39003, datanodeUuid=e3eba06e-e494-45bd-a171-1b16a8edcca7, infoPort=45573, infoSecurePort=0, ipcPort=46113, storageInfo=lv=-57;cid=testClusterID;nsid=612618959;c=1732422575571):Failed to transfer BP-1909738581-172.17.0.2-1732422575571:blk_1073741829_1005 to 127.0.0.1:35379 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T04:30:12,870 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@3e9473f4[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:39003, datanodeUuid=e3eba06e-e494-45bd-a171-1b16a8edcca7, infoPort=45573, infoSecurePort=0, ipcPort=46113, storageInfo=lv=-57;cid=testClusterID;nsid=612618959;c=1732422575571):Failed to transfer BP-1909738581-172.17.0.2-1732422575571:blk_1073741831_1007 to 127.0.0.1:35379 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T04:30:13,617 INFO  [regionserver/fadb2fe0ba32:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:30:14,101 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor fadb2fe0ba32%2C40681%2C1732422576481.1732422614101
2024-11-24T04:30:14,105 WARN  [Thread-1015 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741882_1066
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:35379
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:30:14,105 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-271356777_22 at /127.0.0.1:57008 [Receiving block BP-1909738581-172.17.0.2-1732422575571:blk_1073741882_1066] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/cluster_56272640-4b45-3980-6187-aab4c21513ae/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/cluster_56272640-4b45-3980-6187-aab4c21513ae/data/data10]'}, localName='127.0.0.1:45685', datanodeUuid='a280f544-a74c-41cc-82b9-31a5022433e7', xmitsInProgress=0}:Exception transferring block BP-1909738581-172.17.0.2-1732422575571:blk_1073741882_1066 to mirror 127.0.0.1:35379
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T04:30:14,105 WARN  [Thread-1015 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1909738581-172.17.0.2-1732422575571:blk_1073741882_1066 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45685,DS-48952f18-74aa-43bd-ad38-e5a4deb9635a,DISK], DatanodeInfoWithStorage[127.0.0.1:35379,DS-4844294c-4a9d-4087-a05b-bdb1c1eed91d,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:35379,DS-4844294c-4a9d-4087-a05b-bdb1c1eed91d,DISK]) is bad.
2024-11-24T04:30:14,105 WARN  [Thread-1015 {}] hdfs.DataStreamer(1850): Abandoning BP-1909738581-172.17.0.2-1732422575571:blk_1073741882_1066
2024-11-24T04:30:14,105 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-271356777_22 at /127.0.0.1:57008 [Receiving block BP-1909738581-172.17.0.2-1732422575571:blk_1073741882_1066] {}] datanode.BlockReceiver(316): Block 1073741882 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-24T04:30:14,105 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-271356777_22 at /127.0.0.1:57008 [Receiving block BP-1909738581-172.17.0.2-1732422575571:blk_1073741882_1066] {}] datanode.DataXceiver(331): 127.0.0.1:45685:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:57008 dst: /127.0.0.1:45685
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T04:30:14,106 WARN  [Thread-1015 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35379,DS-4844294c-4a9d-4087-a05b-bdb1c1eed91d,DISK]
2024-11-24T04:30:14,110 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:14,110 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:14,110 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:14,110 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:14,110 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:14,110 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.1732422596521 with entries=14, filesize=12.95 KB; new WAL /user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.1732422614101
2024-11-24T04:30:14,111 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33121:33121),(127.0.0.1/127.0.0.1:45573:45573)]
2024-11-24T04:30:14,111 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.1732422596521 is not closed yet, will try archiving it next time
2024-11-24T04:30:14,112 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45685 is added to blk_1073741869_1052 (size=13268)
2024-11-24T04:30:14,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40681 {}] regionserver.HRegion(8855): Flush requested on 0ce341802081e9b50470c63603e22c62
2024-11-24T04:30:14,118 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0ce341802081e9b50470c63603e22c62 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-11-24T04:30:14,124 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/.tmp/info/ecdc5406ee524630906e73212094af2a is 1080, key is row0013/info:/1732422614113/Put/seqid=0
2024-11-24T04:30:14,127 WARN  [Thread-1022 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741884_1068
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:35379
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:30:14,127 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_620784384_22 at /127.0.0.1:45242 [Receiving block BP-1909738581-172.17.0.2-1732422575571:blk_1073741884_1068] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/cluster_56272640-4b45-3980-6187-aab4c21513ae/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/cluster_56272640-4b45-3980-6187-aab4c21513ae/data/data4]'}, localName='127.0.0.1:39003', datanodeUuid='e3eba06e-e494-45bd-a171-1b16a8edcca7', xmitsInProgress=0}:Exception transferring block BP-1909738581-172.17.0.2-1732422575571:blk_1073741884_1068 to mirror 127.0.0.1:35379
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T04:30:14,127 WARN  [Thread-1022 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1909738581-172.17.0.2-1732422575571:blk_1073741884_1068 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39003,DS-1c26aeaf-2106-4f04-a38e-568cd2360db7,DISK], DatanodeInfoWithStorage[127.0.0.1:35379,DS-4844294c-4a9d-4087-a05b-bdb1c1eed91d,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:35379,DS-4844294c-4a9d-4087-a05b-bdb1c1eed91d,DISK]) is bad.
2024-11-24T04:30:14,127 WARN  [Thread-1022 {}] hdfs.DataStreamer(1850): Abandoning BP-1909738581-172.17.0.2-1732422575571:blk_1073741884_1068
2024-11-24T04:30:14,127 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_620784384_22 at /127.0.0.1:45242 [Receiving block BP-1909738581-172.17.0.2-1732422575571:blk_1073741884_1068] {}] datanode.BlockReceiver(316): Block 1073741884 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-24T04:30:14,127 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_620784384_22 at /127.0.0.1:45242 [Receiving block BP-1909738581-172.17.0.2-1732422575571:blk_1073741884_1068] {}] datanode.DataXceiver(331): 127.0.0.1:39003:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:45242 dst: /127.0.0.1:39003
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T04:30:14,128 WARN  [Thread-1022 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35379,DS-4844294c-4a9d-4087-a05b-bdb1c1eed91d,DISK]
2024-11-24T04:30:14,133 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45685 is added to blk_1073741885_1069 (size=9267)
2024-11-24T04:30:14,133 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39003 is added to blk_1073741885_1069 (size=9267)
2024-11-24T04:30:14,134 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/.tmp/info/ecdc5406ee524630906e73212094af2a
2024-11-24T04:30:14,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/.tmp/info/ecdc5406ee524630906e73212094af2a as hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/info/ecdc5406ee524630906e73212094af2a
2024-11-24T04:30:14,147 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/info/ecdc5406ee524630906e73212094af2a, entries=4, sequenceid=53, filesize=9.0 K
2024-11-24T04:30:14,149 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7528, heapSize ~8.11 KB/8304, currentSize=8.41 KB/8607 for 0ce341802081e9b50470c63603e22c62 in 30ms, sequenceid=53, compaction requested=true
2024-11-24T04:30:14,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0ce341802081e9b50470c63603e22c62:

2024-11-24T04:30:14,149 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=32.5 K, sizeToCheck=16.0 K
2024-11-24T04:30:14,149 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-24T04:30:14,149 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/info/455b976c3f7d429793f7d5e97510eb27 because midkey is the same as first or last row
2024-11-24T04:30:14,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ce341802081e9b50470c63603e22c62:info, priority=-2147483648, current under compaction store size is 1
2024-11-24T04:30:14,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-24T04:30:14,149 DEBUG [RS:0;fadb2fe0ba32:40681-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-24T04:30:14,150 DEBUG [RS:0;fadb2fe0ba32:40681-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 33288 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-24T04:30:14,150 DEBUG [RS:0;fadb2fe0ba32:40681-shortCompactions-0 {}] regionserver.HStore(1541): 0ce341802081e9b50470c63603e22c62/info is initiating minor compaction (all files)
2024-11-24T04:30:14,150 INFO  [RS:0;fadb2fe0ba32:40681-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ce341802081e9b50470c63603e22c62/info in TestLogRolling-testLogRollOnDatanodeDeath,,1732422577648.0ce341802081e9b50470c63603e22c62.
2024-11-24T04:30:14,151 INFO  [RS:0;fadb2fe0ba32:40681-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/info/455b976c3f7d429793f7d5e97510eb27, hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/info/c98d097882b5455483624014a7841a3f, hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/info/ecdc5406ee524630906e73212094af2a] into tmpdir=hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/.tmp, totalSize=32.5 K
2024-11-24T04:30:14,151 DEBUG [RS:0;fadb2fe0ba32:40681-shortCompactions-0 {}] compactions.Compactor(225): Compacting 455b976c3f7d429793f7d5e97510eb27, keycount=12, bloomtype=ROW, size=17.6 K, encoding=NONE, compression=NONE, seqNum=32, earliestPutTs=1732422590472
2024-11-24T04:30:14,152 DEBUG [RS:0;fadb2fe0ba32:40681-shortCompactions-0 {}] compactions.Compactor(225): Compacting c98d097882b5455483624014a7841a3f, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1732422597550
2024-11-24T04:30:14,152 DEBUG [RS:0;fadb2fe0ba32:40681-shortCompactions-0 {}] compactions.Compactor(225): Compacting ecdc5406ee524630906e73212094af2a, keycount=4, bloomtype=ROW, size=9.0 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732422598362
2024-11-24T04:30:14,169 INFO  [RS:0;fadb2fe0ba32:40681-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ce341802081e9b50470c63603e22c62#info#compaction#24 average throughput is 5.13 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-24T04:30:14,170 DEBUG [RS:0;fadb2fe0ba32:40681-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/.tmp/info/7089a6deea6d4adbbd04aadcd0227f51 is 1080, key is row0002/info:/1732422590472/Put/seqid=0
2024-11-24T04:30:14,171 WARN  [Thread-1032 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741886_1070
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:30:14,172 WARN  [Thread-1032 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1909738581-172.17.0.2-1732422575571:blk_1073741886_1070 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35379,DS-4844294c-4a9d-4087-a05b-bdb1c1eed91d,DISK], DatanodeInfoWithStorage[127.0.0.1:45685,DS-48952f18-74aa-43bd-ad38-e5a4deb9635a,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35379,DS-4844294c-4a9d-4087-a05b-bdb1c1eed91d,DISK]) is bad.
2024-11-24T04:30:14,172 WARN  [Thread-1032 {}] hdfs.DataStreamer(1850): Abandoning BP-1909738581-172.17.0.2-1732422575571:blk_1073741886_1070
2024-11-24T04:30:14,172 WARN  [Thread-1032 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35379,DS-4844294c-4a9d-4087-a05b-bdb1c1eed91d,DISK]
2024-11-24T04:30:14,176 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45685 is added to blk_1073741887_1071 (size=21344)
2024-11-24T04:30:14,177 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39003 is added to blk_1073741887_1071 (size=21344)
2024-11-24T04:30:14,183 DEBUG [RS:0;fadb2fe0ba32:40681-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/.tmp/info/7089a6deea6d4adbbd04aadcd0227f51 as hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/info/7089a6deea6d4adbbd04aadcd0227f51
2024-11-24T04:30:14,190 INFO  [RS:0;fadb2fe0ba32:40681-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0ce341802081e9b50470c63603e22c62/info of 0ce341802081e9b50470c63603e22c62 into 7089a6deea6d4adbbd04aadcd0227f51(size=20.8 K), total size for store is 20.8 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-24T04:30:14,190 DEBUG [RS:0;fadb2fe0ba32:40681-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ce341802081e9b50470c63603e22c62:

2024-11-24T04:30:14,190 INFO  [RS:0;fadb2fe0ba32:40681-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1732422577648.0ce341802081e9b50470c63603e22c62., storeName=0ce341802081e9b50470c63603e22c62/info, priority=13, startTime=1732422614149; duration=0sec
2024-11-24T04:30:14,190 DEBUG [RS:0;fadb2fe0ba32:40681-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=20.8 K, sizeToCheck=16.0 K
2024-11-24T04:30:14,190 DEBUG [RS:0;fadb2fe0ba32:40681-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-24T04:30:14,190 DEBUG [RS:0;fadb2fe0ba32:40681-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/info/7089a6deea6d4adbbd04aadcd0227f51 because midkey is the same as first or last row
2024-11-24T04:30:14,191 DEBUG [RS:0;fadb2fe0ba32:40681-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=20.8 K, sizeToCheck=16.0 K
2024-11-24T04:30:14,191 DEBUG [RS:0;fadb2fe0ba32:40681-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-24T04:30:14,191 DEBUG [RS:0;fadb2fe0ba32:40681-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/info/7089a6deea6d4adbbd04aadcd0227f51 because midkey is the same as first or last row
2024-11-24T04:30:14,191 DEBUG [RS:0;fadb2fe0ba32:40681-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=20.8 K, sizeToCheck=16.0 K
2024-11-24T04:30:14,191 DEBUG [RS:0;fadb2fe0ba32:40681-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-24T04:30:14,191 DEBUG [RS:0;fadb2fe0ba32:40681-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/info/7089a6deea6d4adbbd04aadcd0227f51 because midkey is the same as first or last row
2024-11-24T04:30:14,191 DEBUG [RS:0;fadb2fe0ba32:40681-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-24T04:30:14,191 DEBUG [RS:0;fadb2fe0ba32:40681-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ce341802081e9b50470c63603e22c62:info
2024-11-24T04:30:14,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40681 {}] regionserver.HRegion(8855): Flush requested on 0ce341802081e9b50470c63603e22c62
2024-11-24T04:30:14,338 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0ce341802081e9b50470c63603e22c62 1/1 column families, dataSize=9.46 KB heapSize=10.38 KB
2024-11-24T04:30:14,343 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/.tmp/info/e5a7020b19ca428ab359b9196db81226 is 1080, key is row0016/info:/1732422614119/Put/seqid=0
2024-11-24T04:30:14,345 WARN  [Thread-1038 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741888_1072
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:30:14,345 WARN  [Thread-1038 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1909738581-172.17.0.2-1732422575571:blk_1073741888_1072 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35379,DS-4844294c-4a9d-4087-a05b-bdb1c1eed91d,DISK], DatanodeInfoWithStorage[127.0.0.1:39003,DS-1c26aeaf-2106-4f04-a38e-568cd2360db7,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35379,DS-4844294c-4a9d-4087-a05b-bdb1c1eed91d,DISK]) is bad.
2024-11-24T04:30:14,345 WARN  [Thread-1038 {}] hdfs.DataStreamer(1850): Abandoning BP-1909738581-172.17.0.2-1732422575571:blk_1073741888_1072
2024-11-24T04:30:14,346 WARN  [Thread-1038 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35379,DS-4844294c-4a9d-4087-a05b-bdb1c1eed91d,DISK]
2024-11-24T04:30:14,350 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45685 is added to blk_1073741889_1073 (size=13583)
2024-11-24T04:30:14,350 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39003 is added to blk_1073741889_1073 (size=13583)
2024-11-24T04:30:14,513 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.1732422596521 to hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/oldWALs/fadb2fe0ba32%2C40681%2C1732422576481.1732422596521
2024-11-24T04:30:14,540 INFO  [regionserver/fadb2fe0ba32:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:30:14,540 INFO  [regionserver/fadb2fe0ba32:0.logRoller {}] wal.FSHLog(556): LowReplication-Roller was enabled.
2024-11-24T04:30:14,741 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-24T04:30:14,742 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-24T04:30:14,742 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-24T04:30:14,742 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T04:30:14,742 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T04:30:14,742 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-24T04:30:14,742 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-24T04:30:14,742 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1650848494, stopped=false
2024-11-24T04:30:14,743 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=fadb2fe0ba32,35377,1732422576418
2024-11-24T04:30:14,744 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35377-0x101483c93a00000, quorum=127.0.0.1:62127, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-24T04:30:14,744 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44377-0x101483c93a00002, quorum=127.0.0.1:62127, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-24T04:30:14,744 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40681-0x101483c93a00001, quorum=127.0.0.1:62127, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-24T04:30:14,744 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44377-0x101483c93a00002, quorum=127.0.0.1:62127, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:30:14,744 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40681-0x101483c93a00001, quorum=127.0.0.1:62127, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:30:14,745 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35377-0x101483c93a00000, quorum=127.0.0.1:62127, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:30:14,745 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-24T04:30:14,745 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-24T04:30:14,745 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44377-0x101483c93a00002, quorum=127.0.0.1:62127, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-24T04:30:14,745 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40681-0x101483c93a00001, quorum=127.0.0.1:62127, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-24T04:30:14,745 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35377-0x101483c93a00000, quorum=127.0.0.1:62127, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-24T04:30:14,745 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-24T04:30:14,745 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T04:30:14,746 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'fadb2fe0ba32,40681,1732422576481' *****
2024-11-24T04:30:14,746 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-24T04:30:14,746 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'fadb2fe0ba32,44377,1732422577562' *****
2024-11-24T04:30:14,746 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-24T04:30:14,746 INFO  [RS:0;fadb2fe0ba32:40681 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-24T04:30:14,747 INFO  [RS:1;fadb2fe0ba32:44377 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-24T04:30:14,747 INFO  [RS:1;fadb2fe0ba32:44377 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-24T04:30:14,747 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-24T04:30:14,747 INFO  [RS:1;fadb2fe0ba32:44377 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-24T04:30:14,747 INFO  [RS:1;fadb2fe0ba32:44377 {}] regionserver.HRegionServer(959): stopping server fadb2fe0ba32,44377,1732422577562
2024-11-24T04:30:14,747 INFO  [RS:1;fadb2fe0ba32:44377 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-24T04:30:14,747 INFO  [RS:1;fadb2fe0ba32:44377 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;fadb2fe0ba32:44377.
2024-11-24T04:30:14,747 DEBUG [RS:1;fadb2fe0ba32:44377 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-24T04:30:14,747 DEBUG [RS:1;fadb2fe0ba32:44377 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T04:30:14,747 INFO  [RS:1;fadb2fe0ba32:44377 {}] regionserver.HRegionServer(976): stopping server fadb2fe0ba32,44377,1732422577562; all regions closed.
2024-11-24T04:30:14,748 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:14,748 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:14,748 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:14,748 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:14,748 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:14,749 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:30:14,749 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:30:14,749 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
2024-11-24T04:30:14,750 WARN  [IPC Server handler 1 on default port 40187 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749 has not been closed. Lease recovery is in progress. RecoveryId = 1074 for block blk_1073741837_1013
2024-11-24T04:30:14,750 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749 after 1ms
2024-11-24T04:30:14,751 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=9.46 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/.tmp/info/e5a7020b19ca428ab359b9196db81226
2024-11-24T04:30:14,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/.tmp/info/e5a7020b19ca428ab359b9196db81226 as hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/info/e5a7020b19ca428ab359b9196db81226
2024-11-24T04:30:14,765 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/info/e5a7020b19ca428ab359b9196db81226, entries=8, sequenceid=66, filesize=13.3 K
2024-11-24T04:30:14,767 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~9.46 KB/9682, heapSize ~10.36 KB/10608, currentSize=1.05 KB/1075 for 0ce341802081e9b50470c63603e22c62 in 428ms, sequenceid=66, compaction requested=false
2024-11-24T04:30:14,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0ce341802081e9b50470c63603e22c62:

2024-11-24T04:30:14,767 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=34.1 K, sizeToCheck=16.0 K
2024-11-24T04:30:14,767 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-24T04:30:14,767 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/info/7089a6deea6d4adbbd04aadcd0227f51 because midkey is the same as first or last row
2024-11-24T04:30:14,767 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-24T04:30:14,767 INFO  [RS:0;fadb2fe0ba32:40681 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-24T04:30:14,767 INFO  [RS:0;fadb2fe0ba32:40681 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-24T04:30:14,767 INFO  [RS:0;fadb2fe0ba32:40681 {}] regionserver.HRegionServer(3091): Received CLOSE for 0ce341802081e9b50470c63603e22c62
2024-11-24T04:30:14,768 INFO  [RS:0;fadb2fe0ba32:40681 {}] regionserver.HRegionServer(959): stopping server fadb2fe0ba32,40681,1732422576481
2024-11-24T04:30:14,768 INFO  [RS:0;fadb2fe0ba32:40681 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-24T04:30:14,768 INFO  [RS:0;fadb2fe0ba32:40681 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;fadb2fe0ba32:40681.
2024-11-24T04:30:14,768 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0ce341802081e9b50470c63603e22c62, disabling compactions & flushes
2024-11-24T04:30:14,768 DEBUG [RS:0;fadb2fe0ba32:40681 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-24T04:30:14,768 DEBUG [RS:0;fadb2fe0ba32:40681 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T04:30:14,768 INFO  [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1732422577648.0ce341802081e9b50470c63603e22c62.
2024-11-24T04:30:14,768 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1732422577648.0ce341802081e9b50470c63603e22c62.
2024-11-24T04:30:14,768 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1732422577648.0ce341802081e9b50470c63603e22c62. after waiting 0 ms
2024-11-24T04:30:14,768 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1732422577648.0ce341802081e9b50470c63603e22c62.
2024-11-24T04:30:14,768 INFO  [RS:0;fadb2fe0ba32:40681 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-24T04:30:14,768 INFO  [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 0ce341802081e9b50470c63603e22c62 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-24T04:30:14,768 INFO  [RS:0;fadb2fe0ba32:40681 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-24T04:30:14,768 INFO  [RS:0;fadb2fe0ba32:40681 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-24T04:30:14,768 INFO  [RS:0;fadb2fe0ba32:40681 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-24T04:30:14,769 INFO  [RS:0;fadb2fe0ba32:40681 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-11-24T04:30:14,769 DEBUG [RS:0;fadb2fe0ba32:40681 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 0ce341802081e9b50470c63603e22c62=TestLogRolling-testLogRollOnDatanodeDeath,,1732422577648.0ce341802081e9b50470c63603e22c62.}
2024-11-24T04:30:14,769 DEBUG [RS:0;fadb2fe0ba32:40681 {}] regionserver.HRegionServer(1351): Waiting on 0ce341802081e9b50470c63603e22c62, 1588230740
2024-11-24T04:30:14,769 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-24T04:30:14,769 INFO  [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-24T04:30:14,769 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-24T04:30:14,769 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-24T04:30:14,769 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-24T04:30:14,769 INFO  [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.71 KB heapSize=3.75 KB
2024-11-24T04:30:14,769 ERROR [FSHLog-0-hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc-prefix:fadb2fe0ba32,40681,1732422576481.meta {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:30:14,770 WARN  [FSHLog-0-hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc-prefix:fadb2fe0ba32,40681,1732422576481.meta {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:30:14,770 DEBUG [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog fadb2fe0ba32%2C40681%2C1732422576481.meta:.meta(num 1732422577361) roll requested
2024-11-24T04:30:14,770 INFO  [regionserver/fadb2fe0ba32:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422614770.meta
2024-11-24T04:30:14,773 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/.tmp/info/ed99469ec9664186a525ef2bc906baf2 is 1079, key is tmprow/info:/1732422614540/Put/seqid=0
2024-11-24T04:30:14,773 WARN  [Thread-1044 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741890_1075
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:30:14,773 WARN  [Thread-1044 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1909738581-172.17.0.2-1732422575571:blk_1073741890_1075 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35379,DS-4844294c-4a9d-4087-a05b-bdb1c1eed91d,DISK], DatanodeInfoWithStorage[127.0.0.1:45685,DS-48952f18-74aa-43bd-ad38-e5a4deb9635a,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35379,DS-4844294c-4a9d-4087-a05b-bdb1c1eed91d,DISK]) is bad.
2024-11-24T04:30:14,774 WARN  [Thread-1044 {}] hdfs.DataStreamer(1850): Abandoning BP-1909738581-172.17.0.2-1732422575571:blk_1073741890_1075
2024-11-24T04:30:14,775 WARN  [Thread-1044 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35379,DS-4844294c-4a9d-4087-a05b-bdb1c1eed91d,DISK]
2024-11-24T04:30:14,786 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39003 is added to blk_1073741891_1076 (size=6027)
2024-11-24T04:30:14,786 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45685 is added to blk_1073741891_1076 (size=6027)
2024-11-24T04:30:14,787 INFO  [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/.tmp/info/ed99469ec9664186a525ef2bc906baf2
2024-11-24T04:30:14,794 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/.tmp/info/ed99469ec9664186a525ef2bc906baf2 as hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/info/ed99469ec9664186a525ef2bc906baf2
2024-11-24T04:30:14,795 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:14,795 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:14,795 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:14,795 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:14,795 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:14,796 INFO  [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta with entries=8, filesize=2.33 KB; new WAL /user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422614770.meta
2024-11-24T04:30:14,796 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:30:14,796 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46879,DS-ebb5afcd-915f-47cc-8761-64b36c291deb,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:30:14,796 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
2024-11-24T04:30:14,797 WARN  [IPC Server handler 2 on default port 40187 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta has not been closed. Lease recovery is in progress. RecoveryId = 1078 for block blk_1073741834_1010
2024-11-24T04:30:14,797 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta after 1ms
2024-11-24T04:30:14,801 INFO  [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/info/ed99469ec9664186a525ef2bc906baf2, entries=1, sequenceid=70, filesize=5.9 K
2024-11-24T04:30:14,801 DEBUG [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45573:45573),(127.0.0.1/127.0.0.1:33121:33121)]
2024-11-24T04:30:14,801 DEBUG [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta is not closed yet, will try archiving it next time
2024-11-24T04:30:14,802 INFO  [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1075, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 0ce341802081e9b50470c63603e22c62 in 34ms, sequenceid=70, compaction requested=true
2024-11-24T04:30:14,803 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732422577648.0ce341802081e9b50470c63603e22c62.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/info/b81a18220de8466091d6f09693816a0b, hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/info/33e16ffd3dfd40b7be285078a5953437, hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/info/455b976c3f7d429793f7d5e97510eb27, hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/info/d6b5a3734c0e4e0495e0d804f22d7ae4, hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/info/c98d097882b5455483624014a7841a3f, hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/info/ecdc5406ee524630906e73212094af2a] to archive
2024-11-24T04:30:14,804 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732422577648.0ce341802081e9b50470c63603e22c62.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-24T04:30:14,806 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732422577648.0ce341802081e9b50470c63603e22c62.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/info/b81a18220de8466091d6f09693816a0b to hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/info/b81a18220de8466091d6f09693816a0b
2024-11-24T04:30:14,808 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732422577648.0ce341802081e9b50470c63603e22c62.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/info/33e16ffd3dfd40b7be285078a5953437 to hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/info/33e16ffd3dfd40b7be285078a5953437
2024-11-24T04:30:14,810 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732422577648.0ce341802081e9b50470c63603e22c62.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/info/455b976c3f7d429793f7d5e97510eb27 to hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/info/455b976c3f7d429793f7d5e97510eb27
2024-11-24T04:30:14,812 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732422577648.0ce341802081e9b50470c63603e22c62.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/info/d6b5a3734c0e4e0495e0d804f22d7ae4 to hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/info/d6b5a3734c0e4e0495e0d804f22d7ae4
2024-11-24T04:30:14,813 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732422577648.0ce341802081e9b50470c63603e22c62.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/info/c98d097882b5455483624014a7841a3f to hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/info/c98d097882b5455483624014a7841a3f
2024-11-24T04:30:14,815 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732422577648.0ce341802081e9b50470c63603e22c62.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/info/ecdc5406ee524630906e73212094af2a to hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/info/ecdc5406ee524630906e73212094af2a
2024-11-24T04:30:14,815 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732422577648.0ce341802081e9b50470c63603e22c62.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=fadb2fe0ba32:35377 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 16 more
2024-11-24T04:30:14,815 WARN  [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732422577648.0ce341802081e9b50470c63603e22c62.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [b81a18220de8466091d6f09693816a0b=10347, 33e16ffd3dfd40b7be285078a5953437=12506, 455b976c3f7d429793f7d5e97510eb27=17994, d6b5a3734c0e4e0495e0d804f22d7ae4=6027, c98d097882b5455483624014a7841a3f=6027, ecdc5406ee524630906e73212094af2a=9267]
2024-11-24T04:30:14,821 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/default/TestLogRolling-testLogRollOnDatanodeDeath/0ce341802081e9b50470c63603e22c62/recovered.edits/73.seqid, newMaxSeqId=73, maxSeqId=1
2024-11-24T04:30:14,822 INFO  [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1732422577648.0ce341802081e9b50470c63603e22c62.
2024-11-24T04:30:14,822 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0ce341802081e9b50470c63603e22c62:
Waiting for close lock at 1732422614768Running coprocessor pre-close hooks at 1732422614768Disabling compacts and flushes for region at 1732422614768Disabling writes for close at 1732422614768Obtaining lock to block concurrent updates at 1732422614768Preparing flush snapshotting stores in 0ce341802081e9b50470c63603e22c62 at 1732422614768Finished memstore snapshotting TestLogRolling-testLogRollOnDatanodeDeath,,1732422577648.0ce341802081e9b50470c63603e22c62., syncing WAL and waiting on mvcc, flushsize=dataSize=1075, getHeapSize=1392, getOffHeapSize=0, getCellsCount=1 at 1732422614768Flushing stores of TestLogRolling-testLogRollOnDatanodeDeath,,1732422577648.0ce341802081e9b50470c63603e22c62. at 1732422614769 (+1 ms)Flushing 0ce341802081e9b50470c63603e22c62/info: creating writer at 1732422614769Flushing 0ce341802081e9b50470c63603e22c62/info: appending metadata at 1732422614773 (+4 ms)Flushing 0ce341802081e9b50470c63603e22c62/info: closing flushed file at 1732422614773Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@56e52983: reopening flushed file at 1732422614793 (+20 ms)Finished flush of dataSize ~1.05 KB/1075, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 0ce341802081e9b50470c63603e22c62 in 34ms, sequenceid=70, compaction requested=true at 1732422614802 (+9 ms)Writing region close event to WAL at 1732422614816 (+14 ms)Running coprocessor post-close hooks at 1732422614821 (+5 ms)Closed at 1732422614822 (+1 ms)
2024-11-24T04:30:14,822 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1732422577648.0ce341802081e9b50470c63603e22c62.
2024-11-24T04:30:14,825 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/hbase/meta/1588230740/.tmp/info/5f5acb8bdaa74f599374946c97ba6f75 is 203, key is TestLogRolling-testLogRollOnDatanodeDeath,,1732422577648.0ce341802081e9b50470c63603e22c62./info:regioninfo/1732422578009/Put/seqid=0
2024-11-24T04:30:14,827 WARN  [Thread-1056 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741893_1079
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:30:14,827 WARN  [Thread-1056 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1909738581-172.17.0.2-1732422575571:blk_1073741893_1079 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35379,DS-4844294c-4a9d-4087-a05b-bdb1c1eed91d,DISK], DatanodeInfoWithStorage[127.0.0.1:39003,DS-1c26aeaf-2106-4f04-a38e-568cd2360db7,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35379,DS-4844294c-4a9d-4087-a05b-bdb1c1eed91d,DISK]) is bad.
2024-11-24T04:30:14,827 WARN  [Thread-1056 {}] hdfs.DataStreamer(1850): Abandoning BP-1909738581-172.17.0.2-1732422575571:blk_1073741893_1079
2024-11-24T04:30:14,828 WARN  [Thread-1056 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35379,DS-4844294c-4a9d-4087-a05b-bdb1c1eed91d,DISK]
2024-11-24T04:30:14,838 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45685 is added to blk_1073741894_1080 (size=7089)
2024-11-24T04:30:14,841 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39003 is added to blk_1073741894_1080 (size=7089)
2024-11-24T04:30:14,845 INFO  [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.50 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/hbase/meta/1588230740/.tmp/info/5f5acb8bdaa74f599374946c97ba6f75
2024-11-24T04:30:14,855 INFO  [regionserver/fadb2fe0ba32:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-24T04:30:14,875 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/hbase/meta/1588230740/.tmp/ns/79fb1d157b0646728e379f303c375a9b is 43, key is default/ns:d/1732422577427/Put/seqid=0
2024-11-24T04:30:14,877 WARN  [Thread-1063 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741895_1081
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:30:14,877 WARN  [Thread-1063 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1909738581-172.17.0.2-1732422575571:blk_1073741895_1081 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35379,DS-4844294c-4a9d-4087-a05b-bdb1c1eed91d,DISK], DatanodeInfoWithStorage[127.0.0.1:45685,DS-48952f18-74aa-43bd-ad38-e5a4deb9635a,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35379,DS-4844294c-4a9d-4087-a05b-bdb1c1eed91d,DISK]) is bad.
2024-11-24T04:30:14,877 WARN  [Thread-1063 {}] hdfs.DataStreamer(1850): Abandoning BP-1909738581-172.17.0.2-1732422575571:blk_1073741895_1081
2024-11-24T04:30:14,878 WARN  [Thread-1063 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35379,DS-4844294c-4a9d-4087-a05b-bdb1c1eed91d,DISK]
2024-11-24T04:30:14,879 INFO  [regionserver/fadb2fe0ba32:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-11-24T04:30:14,879 INFO  [regionserver/fadb2fe0ba32:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-11-24T04:30:14,886 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39003 is added to blk_1073741896_1082 (size=5153)
2024-11-24T04:30:14,886 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45685 is added to blk_1073741896_1082 (size=5153)
2024-11-24T04:30:14,888 INFO  [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/hbase/meta/1588230740/.tmp/ns/79fb1d157b0646728e379f303c375a9b
2024-11-24T04:30:14,912 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/hbase/meta/1588230740/.tmp/table/331137963d3b443aac1531f3ba6cec08 is 77, key is TestLogRolling-testLogRollOnDatanodeDeath/table:state/1732422578020/Put/seqid=0
2024-11-24T04:30:14,915 WARN  [Thread-1070 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741897_1083
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:30:14,915 WARN  [Thread-1070 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1909738581-172.17.0.2-1732422575571:blk_1073741897_1083 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35379,DS-4844294c-4a9d-4087-a05b-bdb1c1eed91d,DISK], DatanodeInfoWithStorage[127.0.0.1:45685,DS-48952f18-74aa-43bd-ad38-e5a4deb9635a,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35379,DS-4844294c-4a9d-4087-a05b-bdb1c1eed91d,DISK]) is bad.
2024-11-24T04:30:14,915 WARN  [Thread-1070 {}] hdfs.DataStreamer(1850): Abandoning BP-1909738581-172.17.0.2-1732422575571:blk_1073741897_1083
2024-11-24T04:30:14,916 WARN  [Thread-1070 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35379,DS-4844294c-4a9d-4087-a05b-bdb1c1eed91d,DISK]
2024-11-24T04:30:14,921 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45685 is added to blk_1073741898_1084 (size=5424)
2024-11-24T04:30:14,922 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39003 is added to blk_1073741898_1084 (size=5424)
2024-11-24T04:30:14,922 INFO  [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=146 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/hbase/meta/1588230740/.tmp/table/331137963d3b443aac1531f3ba6cec08
2024-11-24T04:30:14,930 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/hbase/meta/1588230740/.tmp/info/5f5acb8bdaa74f599374946c97ba6f75 as hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/hbase/meta/1588230740/info/5f5acb8bdaa74f599374946c97ba6f75
2024-11-24T04:30:14,937 INFO  [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/hbase/meta/1588230740/info/5f5acb8bdaa74f599374946c97ba6f75, entries=10, sequenceid=11, filesize=6.9 K
2024-11-24T04:30:14,937 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/hbase/meta/1588230740/.tmp/ns/79fb1d157b0646728e379f303c375a9b as hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/hbase/meta/1588230740/ns/79fb1d157b0646728e379f303c375a9b
2024-11-24T04:30:14,944 INFO  [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/hbase/meta/1588230740/ns/79fb1d157b0646728e379f303c375a9b, entries=2, sequenceid=11, filesize=5.0 K
2024-11-24T04:30:14,946 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/hbase/meta/1588230740/.tmp/table/331137963d3b443aac1531f3ba6cec08 as hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/hbase/meta/1588230740/table/331137963d3b443aac1531f3ba6cec08
2024-11-24T04:30:14,953 INFO  [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/hbase/meta/1588230740/table/331137963d3b443aac1531f3ba6cec08, entries=2, sequenceid=11, filesize=5.3 K
2024-11-24T04:30:14,954 INFO  [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.71 KB/1752, heapSize ~3.45 KB/3536, currentSize=0 B/0 for 1588230740 in 185ms, sequenceid=11, compaction requested=false
2024-11-24T04:30:14,966 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-11-24T04:30:14,966 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-24T04:30:14,966 INFO  [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-24T04:30:14,966 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732422614769Running coprocessor pre-close hooks at 1732422614769Disabling compacts and flushes for region at 1732422614769Disabling writes for close at 1732422614769Obtaining lock to block concurrent updates at 1732422614769Preparing flush snapshotting stores in 1588230740 at 1732422614769Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1752, getHeapSize=3776, getOffHeapSize=0, getCellsCount=14 at 1732422614769Flushing stores of hbase:meta,,1.1588230740 at 1732422614802 (+33 ms)Flushing 1588230740/info: creating writer at 1732422614802Flushing 1588230740/info: appending metadata at 1732422614824 (+22 ms)Flushing 1588230740/info: closing flushed file at 1732422614824Flushing 1588230740/ns: creating writer at 1732422614852 (+28 ms)Flushing 1588230740/ns: appending metadata at 1732422614874 (+22 ms)Flushing 1588230740/ns: closing flushed file at 1732422614874Flushing 1588230740/table: creating writer at 1732422614895 (+21 ms)Flushing 1588230740/table: appending metadata at 1732422614912 (+17 ms)Flushing 1588230740/table: closing flushed file at 1732422614912Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@17513069: reopening flushed file at 1732422614929 (+17 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4d2dc822: reopening flushed file at 1732422614937 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37a49373: reopening flushed file at 1732422614945 (+8 ms)Finished flush of dataSize ~1.71 KB/1752, heapSize ~3.45 KB/3536, currentSize=0 B/0 for 1588230740 in 185ms, sequenceid=11, compaction requested=false at 1732422614954 (+9 ms)Writing region close event to WAL at 1732422614962 (+8 ms)Running coprocessor post-close hooks at 1732422614966 (+4 ms)Closed at 1732422614966
2024-11-24T04:30:14,967 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-24T04:30:14,969 INFO  [RS:0;fadb2fe0ba32:40681 {}] regionserver.HRegionServer(976): stopping server fadb2fe0ba32,40681,1732422576481; all regions closed.
2024-11-24T04:30:14,969 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:14,969 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:14,970 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:14,970 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:14,970 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:14,972 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45685 is added to blk_1073741892_1077 (size=825)
2024-11-24T04:30:14,972 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39003 is added to blk_1073741892_1077 (size=825)
2024-11-24T04:30:15,425 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39003 is added to blk_1073741869_1052 (size=13268)
2024-11-24T04:30:15,613 INFO  [regionserver/fadb2fe0ba32:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-24T04:30:15,628 INFO  [regionserver/fadb2fe0ba32:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-11-24T04:30:15,628 INFO  [regionserver/fadb2fe0ba32:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-11-24T04:30:15,871 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@5c8c7869[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:39003, datanodeUuid=e3eba06e-e494-45bd-a171-1b16a8edcca7, infoPort=45573, infoSecurePort=0, ipcPort=46113, storageInfo=lv=-57;cid=testClusterID;nsid=612618959;c=1732422575571):Failed to transfer BP-1909738581-172.17.0.2-1732422575571:blk_1073741832_1008 to 127.0.0.1:35379 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T04:30:15,873 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45685 is added to blk_1073741836_1012 (size=76)
2024-11-24T04:30:16,266 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath
2024-11-24T04:30:16,267 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-24T04:30:16,267 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-24T04:30:16,872 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@5c8c7869[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:39003, datanodeUuid=e3eba06e-e494-45bd-a171-1b16a8edcca7, infoPort=45573, infoSecurePort=0, ipcPort=46113, storageInfo=lv=-57;cid=testClusterID;nsid=612618959;c=1732422575571):Failed to transfer BP-1909738581-172.17.0.2-1732422575571:blk_1073741826_1002 to 127.0.0.1:35379 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T04:30:16,874 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45685 is added to blk_1073741828_1004 (size=1189)
2024-11-24T04:30:17,874 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45685 is added to blk_1073741825_1001 (size=7)
2024-11-24T04:30:17,875 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45685 is added to blk_1073741827_1003 (size=196)
2024-11-24T04:30:17,918 INFO  [master/fadb2fe0ba32:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-11-24T04:30:17,918 INFO  [master/fadb2fe0ba32:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-11-24T04:30:18,751 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749 after 4002ms
2024-11-24T04:30:18,798 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta after 4002ms
2024-11-24T04:30:18,901 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@46598435 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1909738581-172.17.0.2-1732422575571:blk_1073741830_1006, datanode=DatanodeInfoWithStorage[127.0.0.1:46879,null,null])
java.net.ConnectException: Call From fadb2fe0ba32/172.17.0.2 to localhost:46669 failed on connection exception: java.net.ConnectException: Connection refused; For more details see:  http://wiki.apache.org/hadoop/ConnectionRefused
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-11-24T04:30:19,749 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds"
2024-11-24T04:30:19,751 DEBUG [RS:1;fadb2fe0ba32:44377 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/oldWALs
2024-11-24T04:30:19,752 INFO  [RS:1;fadb2fe0ba32:44377 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog fadb2fe0ba32%2C44377%2C1732422577562:(num 1732422577749)
2024-11-24T04:30:19,752 DEBUG [RS:1;fadb2fe0ba32:44377 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T04:30:19,752 INFO  [RS:1;fadb2fe0ba32:44377 {}] regionserver.LeaseManager(133): Closed leases
2024-11-24T04:30:19,752 INFO  [RS:1;fadb2fe0ba32:44377 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-24T04:30:19,752 INFO  [RS:1;fadb2fe0ba32:44377 {}] hbase.ChoreService(370): Chore service for: regionserver/fadb2fe0ba32:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-24T04:30:19,752 INFO  [RS:1;fadb2fe0ba32:44377 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-24T04:30:19,752 INFO  [RS:1;fadb2fe0ba32:44377 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-24T04:30:19,752 INFO  [RS:1;fadb2fe0ba32:44377 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-24T04:30:19,752 INFO  [RS:1;fadb2fe0ba32:44377 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-24T04:30:19,752 INFO  [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-24T04:30:19,752 INFO  [RS:1;fadb2fe0ba32:44377 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44377
2024-11-24T04:30:19,754 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35377-0x101483c93a00000, quorum=127.0.0.1:62127, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-24T04:30:19,754 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44377-0x101483c93a00002, quorum=127.0.0.1:62127, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/fadb2fe0ba32,44377,1732422577562
2024-11-24T04:30:19,754 INFO  [RS:1;fadb2fe0ba32:44377 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-24T04:30:19,755 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [fadb2fe0ba32,44377,1732422577562]
2024-11-24T04:30:19,756 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/fadb2fe0ba32,44377,1732422577562 already deleted, retry=false
2024-11-24T04:30:19,756 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; fadb2fe0ba32,44377,1732422577562 expired; onlineServers=1
2024-11-24T04:30:19,757 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1812) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy45.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$isFileClosed$57(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.isFileClosed(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor115.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy46.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor115.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor115.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1810) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:19,823 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:30:19,833 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:30:19,834 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:30:19,834 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:30:19,834 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:30:19,835 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:30:19,840 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:30:19,841 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:30:19,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44377-0x101483c93a00002, quorum=127.0.0.1:62127, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-24T04:30:19,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44377-0x101483c93a00002, quorum=127.0.0.1:62127, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-24T04:30:19,856 INFO  [RS:1;fadb2fe0ba32:44377 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-24T04:30:19,856 INFO  [RS:1;fadb2fe0ba32:44377 {}] regionserver.HRegionServer(1031): Exiting; stopping=fadb2fe0ba32,44377,1732422577562; zookeeper connection closed.
2024-11-24T04:30:19,856 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7d37e119 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7d37e119
2024-11-24T04:30:19,970 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds"
2024-11-24T04:30:19,974 DEBUG [RS:0;fadb2fe0ba32:40681 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/oldWALs
2024-11-24T04:30:19,974 INFO  [RS:0;fadb2fe0ba32:40681 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog fadb2fe0ba32%2C40681%2C1732422576481.meta:.meta(num 1732422614770)
2024-11-24T04:30:19,974 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:19,974 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:19,974 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:19,974 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:19,975 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:19,976 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39003 is added to blk_1073741883_1067 (size=18156)
2024-11-24T04:30:19,977 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45685 is added to blk_1073741883_1067 (size=18156)
2024-11-24T04:30:19,979 DEBUG [RS:0;fadb2fe0ba32:40681 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/oldWALs
2024-11-24T04:30:19,979 INFO  [RS:0;fadb2fe0ba32:40681 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog fadb2fe0ba32%2C40681%2C1732422576481:(num 1732422614101)
2024-11-24T04:30:19,979 DEBUG [RS:0;fadb2fe0ba32:40681 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T04:30:19,979 INFO  [RS:0;fadb2fe0ba32:40681 {}] regionserver.LeaseManager(133): Closed leases
2024-11-24T04:30:19,979 INFO  [RS:0;fadb2fe0ba32:40681 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-24T04:30:19,979 INFO  [RS:0;fadb2fe0ba32:40681 {}] hbase.ChoreService(370): Chore service for: regionserver/fadb2fe0ba32:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-24T04:30:19,979 INFO  [RS:0;fadb2fe0ba32:40681 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-24T04:30:19,979 INFO  [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-24T04:30:19,979 INFO  [RS:0;fadb2fe0ba32:40681 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40681
2024-11-24T04:30:19,983 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35377-0x101483c93a00000, quorum=127.0.0.1:62127, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-24T04:30:19,983 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40681-0x101483c93a00001, quorum=127.0.0.1:62127, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/fadb2fe0ba32,40681,1732422576481
2024-11-24T04:30:19,983 INFO  [RS:0;fadb2fe0ba32:40681 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-24T04:30:19,985 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [fadb2fe0ba32,40681,1732422576481]
2024-11-24T04:30:19,987 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/fadb2fe0ba32,40681,1732422576481 already deleted, retry=false
2024-11-24T04:30:19,987 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; fadb2fe0ba32,40681,1732422576481 expired; onlineServers=0
2024-11-24T04:30:19,987 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'fadb2fe0ba32,35377,1732422576418' *****
2024-11-24T04:30:19,987 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-24T04:30:19,987 INFO  [M:0;fadb2fe0ba32:35377 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-24T04:30:19,987 INFO  [M:0;fadb2fe0ba32:35377 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-24T04:30:19,987 DEBUG [M:0;fadb2fe0ba32:35377 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-24T04:30:19,987 DEBUG [M:0;fadb2fe0ba32:35377 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-24T04:30:19,987 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-24T04:30:19,987 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster-HFileCleaner.small.0-1732422576755 {}] cleaner.HFileCleaner(306): Exit Thread[master/fadb2fe0ba32:0:becomeActiveMaster-HFileCleaner.small.0-1732422576755,5,FailOnTimeoutGroup]
2024-11-24T04:30:19,987 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster-HFileCleaner.large.0-1732422576755 {}] cleaner.HFileCleaner(306): Exit Thread[master/fadb2fe0ba32:0:becomeActiveMaster-HFileCleaner.large.0-1732422576755,5,FailOnTimeoutGroup]
2024-11-24T04:30:19,987 INFO  [M:0;fadb2fe0ba32:35377 {}] hbase.ChoreService(370): Chore service for: master/fadb2fe0ba32:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-24T04:30:19,987 INFO  [M:0;fadb2fe0ba32:35377 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-24T04:30:19,988 DEBUG [M:0;fadb2fe0ba32:35377 {}] master.HMaster(1795): Stopping service threads
2024-11-24T04:30:19,988 INFO  [M:0;fadb2fe0ba32:35377 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-24T04:30:19,988 INFO  [M:0;fadb2fe0ba32:35377 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-24T04:30:19,988 INFO  [M:0;fadb2fe0ba32:35377 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-24T04:30:19,988 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-24T04:30:19,989 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35377-0x101483c93a00000, quorum=127.0.0.1:62127, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-24T04:30:19,989 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35377-0x101483c93a00000, quorum=127.0.0.1:62127, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:30:19,989 DEBUG [M:0;fadb2fe0ba32:35377 {}] zookeeper.ZKUtil(347): master:35377-0x101483c93a00000, quorum=127.0.0.1:62127, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-24T04:30:19,989 WARN  [M:0;fadb2fe0ba32:35377 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-24T04:30:19,990 INFO  [M:0;fadb2fe0ba32:35377 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/.lastflushedseqids
2024-11-24T04:30:19,995 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39003 is added to blk_1073741899_1085 (size=130)
2024-11-24T04:30:19,995 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45685 is added to blk_1073741899_1085 (size=130)
2024-11-24T04:30:19,995 INFO  [M:0;fadb2fe0ba32:35377 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-24T04:30:19,996 INFO  [M:0;fadb2fe0ba32:35377 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-24T04:30:19,996 DEBUG [M:0;fadb2fe0ba32:35377 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-24T04:30:19,996 INFO  [M:0;fadb2fe0ba32:35377 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T04:30:19,996 DEBUG [M:0;fadb2fe0ba32:35377 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T04:30:19,996 DEBUG [M:0;fadb2fe0ba32:35377 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-24T04:30:19,996 DEBUG [M:0;fadb2fe0ba32:35377 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T04:30:19,996 INFO  [M:0;fadb2fe0ba32:35377 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.26 KB heapSize=29.50 KB
2024-11-24T04:30:20,013 DEBUG [M:0;fadb2fe0ba32:35377 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b342ecd39a8642d190e18c1d6a45390a is 82, key is hbase:meta,,1/info:regioninfo/1732422577406/Put/seqid=0
2024-11-24T04:30:20,018 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39003 is added to blk_1073741900_1086 (size=5672)
2024-11-24T04:30:20,018 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45685 is added to blk_1073741900_1086 (size=5672)
2024-11-24T04:30:20,019 INFO  [M:0;fadb2fe0ba32:35377 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b342ecd39a8642d190e18c1d6a45390a
2024-11-24T04:30:20,040 DEBUG [M:0;fadb2fe0ba32:35377 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f022a0873ffe4533a5beb1880ac15db9 is 775, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732422578024/Put/seqid=0
2024-11-24T04:30:20,045 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39003 is added to blk_1073741901_1087 (size=6256)
2024-11-24T04:30:20,046 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45685 is added to blk_1073741901_1087 (size=6256)
2024-11-24T04:30:20,046 INFO  [M:0;fadb2fe0ba32:35377 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.59 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f022a0873ffe4533a5beb1880ac15db9
2024-11-24T04:30:20,052 INFO  [M:0;fadb2fe0ba32:35377 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f022a0873ffe4533a5beb1880ac15db9
2024-11-24T04:30:20,068 DEBUG [M:0;fadb2fe0ba32:35377 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/10fd4b87ae9b441785db4b564f427cd2 is 69, key is fadb2fe0ba32,40681,1732422576481/rs:state/1732422576823/Put/seqid=0
2024-11-24T04:30:20,073 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39003 is added to blk_1073741902_1088 (size=5224)
2024-11-24T04:30:20,073 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45685 is added to blk_1073741902_1088 (size=5224)
2024-11-24T04:30:20,074 INFO  [M:0;fadb2fe0ba32:35377 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/10fd4b87ae9b441785db4b564f427cd2
2024-11-24T04:30:20,085 INFO  [RS:0;fadb2fe0ba32:40681 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-24T04:30:20,085 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40681-0x101483c93a00001, quorum=127.0.0.1:62127, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-24T04:30:20,085 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40681-0x101483c93a00001, quorum=127.0.0.1:62127, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-24T04:30:20,085 INFO  [RS:0;fadb2fe0ba32:40681 {}] regionserver.HRegionServer(1031): Exiting; stopping=fadb2fe0ba32,40681,1732422576481; zookeeper connection closed.
2024-11-24T04:30:20,085 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6c81ac21 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6c81ac21
2024-11-24T04:30:20,085 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete
2024-11-24T04:30:20,094 DEBUG [M:0;fadb2fe0ba32:35377 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/ede2d48a99824ae78d5bf173c174cc14 is 52, key is load_balancer_on/state:d/1732422577514/Put/seqid=0
2024-11-24T04:30:20,099 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45685 is added to blk_1073741903_1089 (size=5056)
2024-11-24T04:30:20,100 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39003 is added to blk_1073741903_1089 (size=5056)
2024-11-24T04:30:20,100 INFO  [M:0;fadb2fe0ba32:35377 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/ede2d48a99824ae78d5bf173c174cc14
2024-11-24T04:30:20,106 DEBUG [M:0;fadb2fe0ba32:35377 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b342ecd39a8642d190e18c1d6a45390a as hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b342ecd39a8642d190e18c1d6a45390a
2024-11-24T04:30:20,111 INFO  [M:0;fadb2fe0ba32:35377 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b342ecd39a8642d190e18c1d6a45390a, entries=8, sequenceid=60, filesize=5.5 K
2024-11-24T04:30:20,112 DEBUG [M:0;fadb2fe0ba32:35377 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f022a0873ffe4533a5beb1880ac15db9 as hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f022a0873ffe4533a5beb1880ac15db9
2024-11-24T04:30:20,118 INFO  [M:0;fadb2fe0ba32:35377 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f022a0873ffe4533a5beb1880ac15db9
2024-11-24T04:30:20,118 INFO  [M:0;fadb2fe0ba32:35377 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f022a0873ffe4533a5beb1880ac15db9, entries=6, sequenceid=60, filesize=6.1 K
2024-11-24T04:30:20,119 DEBUG [M:0;fadb2fe0ba32:35377 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/10fd4b87ae9b441785db4b564f427cd2 as hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/10fd4b87ae9b441785db4b564f427cd2
2024-11-24T04:30:20,124 INFO  [M:0;fadb2fe0ba32:35377 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/10fd4b87ae9b441785db4b564f427cd2, entries=2, sequenceid=60, filesize=5.1 K
2024-11-24T04:30:20,125 DEBUG [M:0;fadb2fe0ba32:35377 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/ede2d48a99824ae78d5bf173c174cc14 as hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/ede2d48a99824ae78d5bf173c174cc14
2024-11-24T04:30:20,130 INFO  [M:0;fadb2fe0ba32:35377 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/ede2d48a99824ae78d5bf173c174cc14, entries=1, sequenceid=60, filesize=4.9 K
2024-11-24T04:30:20,131 INFO  [M:0;fadb2fe0ba32:35377 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.26 KB/23817, heapSize ~29.44 KB/30144, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 135ms, sequenceid=60, compaction requested=false
2024-11-24T04:30:20,133 INFO  [M:0;fadb2fe0ba32:35377 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T04:30:20,133 DEBUG [M:0;fadb2fe0ba32:35377 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732422619996Disabling compacts and flushes for region at 1732422619996Disabling writes for close at 1732422619996Obtaining lock to block concurrent updates at 1732422619996Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732422619996Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23817, getHeapSize=30144, getOffHeapSize=0, getCellsCount=71 at 1732422619996Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732422619997 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732422619997Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732422620012 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732422620012Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732422620024 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732422620039 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732422620040 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732422620052 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732422620067 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732422620067Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732422620079 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732422620094 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732422620094Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7e62c918: reopening flushed file at 1732422620105 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7da15eca: reopening flushed file at 1732422620112 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c8183b8: reopening flushed file at 1732422620118 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@71d65482: reopening flushed file at 1732422620124 (+6 ms)Finished flush of dataSize ~23.26 KB/23817, heapSize ~29.44 KB/30144, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 135ms, sequenceid=60, compaction requested=false at 1732422620131 (+7 ms)Writing region close event to WAL at 1732422620133 (+2 ms)Closed at 1732422620133
2024-11-24T04:30:20,134 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:20,134 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:20,134 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:20,134 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:20,134 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:20,136 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45685 is added to blk_1073741881_1064 (size=1045)
2024-11-24T04:30:20,136 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39003 is added to blk_1073741881_1064 (size=1045)
2024-11-24T04:30:20,137 INFO  [M:0;fadb2fe0ba32:35377 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-24T04:30:20,137 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-24T04:30:20,137 INFO  [M:0;fadb2fe0ba32:35377 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35377
2024-11-24T04:30:20,138 INFO  [M:0;fadb2fe0ba32:35377 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-24T04:30:20,242 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35377-0x101483c93a00000, quorum=127.0.0.1:62127, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-24T04:30:20,242 INFO  [M:0;fadb2fe0ba32:35377 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-24T04:30:20,242 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35377-0x101483c93a00000, quorum=127.0.0.1:62127, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-24T04:30:20,244 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@c1d8e25{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T04:30:20,245 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5af75f98{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-24T04:30:20,245 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-24T04:30:20,245 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@23df32cf{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-24T04:30:20,245 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@16a2580d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/hadoop.log.dir/,STOPPED}
2024-11-24T04:30:20,246 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-24T04:30:20,246 WARN  [BP-1909738581-172.17.0.2-1732422575571 heartbeating to localhost/127.0.0.1:40187 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-24T04:30:20,246 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-24T04:30:20,246 WARN  [BP-1909738581-172.17.0.2-1732422575571 heartbeating to localhost/127.0.0.1:40187 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1909738581-172.17.0.2-1732422575571 (Datanode Uuid e3eba06e-e494-45bd-a171-1b16a8edcca7) service to localhost/127.0.0.1:40187
2024-11-24T04:30:20,246 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@d842e58 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1909738581-172.17.0.2-1732422575571:blk_1073741837_1013, datanode=DatanodeInfoWithStorage[127.0.0.1:46879,null,null])
java.io.InterruptedIOException: DestHost:destPort localhost:46669 , LocalHost:localPort fadb2fe0ba32/172.17.0.2:0. Failed on local exception: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS)
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:936) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS)
	at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:963) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
Caused by: java.lang.InterruptedException: sleep interrupted
	at java.lang.Thread.sleep(Native Method) ~[?:?]
	at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-11-24T04:30:20,247 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@d842e58 {}] datanode.BlockRecoveryWorker$1(605): recover Block: RecoveringBlock{BP-1909738581-172.17.0.2-1732422575571:blk_1073741837_1013; getBlockSize()=85; corrupt=false; offset=-1; locs=[DatanodeInfoWithStorage[127.0.0.1:39003,null,null], DatanodeInfoWithStorage[127.0.0.1:46879,null,null]]; cachedLocs=[]} FAILED: java.io.IOException: No block pool offer service for bpid=BP-1909738581-172.17.0.2-1732422575571
2024-11-24T04:30:20,247 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@d842e58 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1909738581-172.17.0.2-1732422575571:blk_1073741834_1010, datanode=DatanodeInfoWithStorage[127.0.0.1:39003,null,null])
java.io.IOException: No block pool offer service for bpid=BP-1909738581-172.17.0.2-1732422575571
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.getDatanodeID(BlockRecoveryWorker.java:539) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$000(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:131) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T04:30:20,247 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@d842e58 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1909738581-172.17.0.2-1732422575571:blk_1073741834_1010, datanode=DatanodeInfoWithStorage[127.0.0.1:46879,null,null])
java.io.IOException: No block pool offer service for bpid=BP-1909738581-172.17.0.2-1732422575571
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.getDatanodeID(BlockRecoveryWorker.java:539) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$000(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:131) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T04:30:20,247 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@d842e58 {}] datanode.BlockRecoveryWorker$1(605): recover Block: RecoveringBlock{BP-1909738581-172.17.0.2-1732422575571:blk_1073741834_1010; getBlockSize()=85; corrupt=false; offset=-1; locs=[DatanodeInfoWithStorage[127.0.0.1:39003,null,null], DatanodeInfoWithStorage[127.0.0.1:46879,null,null]]; cachedLocs=[]} FAILED: java.io.IOException: All datanodes failed: block=BP-1909738581-172.17.0.2-1732422575571:blk_1073741834_1010, datanodeids=[DatanodeInfoWithStorage[127.0.0.1:39003,null,null], DatanodeInfoWithStorage[127.0.0.1:46879,null,null]]
2024-11-24T04:30:20,247 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/cluster_56272640-4b45-3980-6187-aab4c21513ae/data/data3/current/BP-1909738581-172.17.0.2-1732422575571 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T04:30:20,247 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/cluster_56272640-4b45-3980-6187-aab4c21513ae/data/data4/current/BP-1909738581-172.17.0.2-1732422575571 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T04:30:20,248 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-24T04:30:20,250 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5ea37f0d{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T04:30:20,250 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@15010086{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-24T04:30:20,250 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-24T04:30:20,250 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@78e445ac{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-24T04:30:20,250 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@68004957{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/hadoop.log.dir/,STOPPED}
2024-11-24T04:30:20,251 WARN  [BP-1909738581-172.17.0.2-1732422575571 heartbeating to localhost/127.0.0.1:40187 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-24T04:30:20,251 WARN  [BP-1909738581-172.17.0.2-1732422575571 heartbeating to localhost/127.0.0.1:40187 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1909738581-172.17.0.2-1732422575571 (Datanode Uuid a280f544-a74c-41cc-82b9-31a5022433e7) service to localhost/127.0.0.1:40187
2024-11-24T04:30:20,252 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/cluster_56272640-4b45-3980-6187-aab4c21513ae/data/data9/current/BP-1909738581-172.17.0.2-1732422575571 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T04:30:20,252 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-24T04:30:20,252 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-24T04:30:20,252 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/cluster_56272640-4b45-3980-6187-aab4c21513ae/data/data10/current/BP-1909738581-172.17.0.2-1732422575571 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T04:30:20,253 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-24T04:30:20,258 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7cd2a640{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-24T04:30:20,259 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@64b7b556{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-24T04:30:20,259 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-24T04:30:20,259 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3ed3a961{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-24T04:30:20,259 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6a488aac{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/hadoop.log.dir/,STOPPED}
2024-11-24T04:30:20,267 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-24T04:30:20,306 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-24T04:30:20,315 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=153 (was 78)
Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-5-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40187
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-5-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-20-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40187
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-5
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:40187 from jenkins.hfs.2
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: Close-WAL-Writer-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$900/0x00007f1d54bf4b78.run(Unknown Source)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-18-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-21-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-18-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-21-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-7-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-19-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:40187 from jenkins.hfs.3
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-19-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:40093
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-20-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-18-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-21-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.3@localhost:40187
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-7-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.1@localhost:40093
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-5-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-19-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Close-WAL-Writer-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$900/0x00007f1d54bf4b78.run(Unknown Source)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:40187 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-20-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.2@localhost:40187
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40187
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-7-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:40187 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40187
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:40187
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=450 (was 402) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=167 (was 148) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6780 (was 7078)
2024-11-24T04:30:20,324 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=153, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=167, ProcessCount=11, AvailableMemoryMB=6779
2024-11-24T04:30:20,324 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-24T04:30:20,324 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/hadoop.log.dir so I do NOT create it in target/test-data/7f4065fa-2df8-4567-fbb0-481a3f88ce22
2024-11-24T04:30:20,324 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ae17b00d-f75b-ab66-f7d2-3461ce964fb1/hadoop.tmp.dir so I do NOT create it in target/test-data/7f4065fa-2df8-4567-fbb0-481a3f88ce22
2024-11-24T04:30:20,324 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f4065fa-2df8-4567-fbb0-481a3f88ce22/cluster_7730fbad-df8e-ea9e-97b3-261c7f308fa8, deleteOnExit=true
2024-11-24T04:30:20,324 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-24T04:30:20,325 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f4065fa-2df8-4567-fbb0-481a3f88ce22/test.cache.data in system properties and HBase conf
2024-11-24T04:30:20,325 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f4065fa-2df8-4567-fbb0-481a3f88ce22/hadoop.tmp.dir in system properties and HBase conf
2024-11-24T04:30:20,325 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f4065fa-2df8-4567-fbb0-481a3f88ce22/hadoop.log.dir in system properties and HBase conf
2024-11-24T04:30:20,325 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f4065fa-2df8-4567-fbb0-481a3f88ce22/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-24T04:30:20,325 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f4065fa-2df8-4567-fbb0-481a3f88ce22/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-24T04:30:20,325 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-24T04:30:20,325 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-24T04:30:20,325 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f4065fa-2df8-4567-fbb0-481a3f88ce22/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-24T04:30:20,325 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f4065fa-2df8-4567-fbb0-481a3f88ce22/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-24T04:30:20,325 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f4065fa-2df8-4567-fbb0-481a3f88ce22/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-24T04:30:20,325 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f4065fa-2df8-4567-fbb0-481a3f88ce22/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-24T04:30:20,326 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f4065fa-2df8-4567-fbb0-481a3f88ce22/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-24T04:30:20,326 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f4065fa-2df8-4567-fbb0-481a3f88ce22/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-24T04:30:20,326 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f4065fa-2df8-4567-fbb0-481a3f88ce22/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-24T04:30:20,326 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f4065fa-2df8-4567-fbb0-481a3f88ce22/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-24T04:30:20,326 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f4065fa-2df8-4567-fbb0-481a3f88ce22/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-24T04:30:20,326 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f4065fa-2df8-4567-fbb0-481a3f88ce22/nfs.dump.dir in system properties and HBase conf
2024-11-24T04:30:20,326 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f4065fa-2df8-4567-fbb0-481a3f88ce22/java.io.tmpdir in system properties and HBase conf
2024-11-24T04:30:20,326 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f4065fa-2df8-4567-fbb0-481a3f88ce22/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-24T04:30:20,326 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f4065fa-2df8-4567-fbb0-481a3f88ce22/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-24T04:30:20,326 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f4065fa-2df8-4567-fbb0-481a3f88ce22/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-24T04:30:20,341 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-24T04:30:20,343 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-24T04:30:20,355 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:30:20,356 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:30:20,356 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:30:20,356 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:30:20,357 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:30:20,357 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:30:20,360 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:30:20,360 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:30:20,360 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:30:20,362 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:30:20,430 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T04:30:20,434 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-24T04:30:20,436 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-24T04:30:20,436 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-24T04:30:20,436 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-24T04:30:20,437 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T04:30:20,437 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2ddfefdc{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f4065fa-2df8-4567-fbb0-481a3f88ce22/hadoop.log.dir/,AVAILABLE}
2024-11-24T04:30:20,437 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@17b2a9ba{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-24T04:30:20,551 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5d7c8143{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f4065fa-2df8-4567-fbb0-481a3f88ce22/java.io.tmpdir/jetty-localhost-40819-hadoop-hdfs-3_4_1-tests_jar-_-any-17911959423283712622/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-24T04:30:20,552 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6de1af65{HTTP/1.1, (http/1.1)}{localhost:40819}
2024-11-24T04:30:20,552 INFO  [Time-limited test {}] server.Server(415): Started @149613ms
2024-11-24T04:30:20,565 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-24T04:30:20,640 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T04:30:20,643 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-24T04:30:20,644 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-24T04:30:20,644 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-24T04:30:20,644 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-24T04:30:20,645 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2b6caabd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f4065fa-2df8-4567-fbb0-481a3f88ce22/hadoop.log.dir/,AVAILABLE}
2024-11-24T04:30:20,645 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2744dc92{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-24T04:30:20,758 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:20,761 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5493c38f{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f4065fa-2df8-4567-fbb0-481a3f88ce22/java.io.tmpdir/jetty-localhost-35519-hadoop-hdfs-3_4_1-tests_jar-_-any-3245200812973937950/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T04:30:20,762 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@50743f5b{HTTP/1.1, (http/1.1)}{localhost:35519}
2024-11-24T04:30:20,762 INFO  [Time-limited test {}] server.Server(415): Started @149824ms
2024-11-24T04:30:20,764 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-24T04:30:20,799 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:20,810 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T04:30:20,813 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-24T04:30:20,817 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-24T04:30:20,817 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-24T04:30:20,817 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-24T04:30:20,818 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7ec5b9af{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f4065fa-2df8-4567-fbb0-481a3f88ce22/hadoop.log.dir/,AVAILABLE}
2024-11-24T04:30:20,818 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@671953b5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-24T04:30:20,882 WARN  [Thread-1191 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f4065fa-2df8-4567-fbb0-481a3f88ce22/cluster_7730fbad-df8e-ea9e-97b3-261c7f308fa8/data/data2/current/BP-878497079-172.17.0.2-1732422620375/current, will proceed with Du for space computation calculation, 
2024-11-24T04:30:20,882 WARN  [Thread-1190 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f4065fa-2df8-4567-fbb0-481a3f88ce22/cluster_7730fbad-df8e-ea9e-97b3-261c7f308fa8/data/data1/current/BP-878497079-172.17.0.2-1732422620375/current, will proceed with Du for space computation calculation, 
2024-11-24T04:30:20,899 WARN  [Thread-1169 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-24T04:30:20,902 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x76243acccb837399 with lease ID 0xff875ab8a6d5170e: Processing first storage report for DS-38ec15dd-c607-46cb-8f20-43e17ecaf8ee from datanode DatanodeRegistration(127.0.0.1:41397, datanodeUuid=5e400a8b-e68b-4fbe-8406-f707b7cad460, infoPort=38915, infoSecurePort=0, ipcPort=36023, storageInfo=lv=-57;cid=testClusterID;nsid=678129203;c=1732422620375)
2024-11-24T04:30:20,902 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x76243acccb837399 with lease ID 0xff875ab8a6d5170e: from storage DS-38ec15dd-c607-46cb-8f20-43e17ecaf8ee node DatanodeRegistration(127.0.0.1:41397, datanodeUuid=5e400a8b-e68b-4fbe-8406-f707b7cad460, infoPort=38915, infoSecurePort=0, ipcPort=36023, storageInfo=lv=-57;cid=testClusterID;nsid=678129203;c=1732422620375), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T04:30:20,902 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x76243acccb837399 with lease ID 0xff875ab8a6d5170e: Processing first storage report for DS-40d508c5-83fc-4808-89de-d8587e88042a from datanode DatanodeRegistration(127.0.0.1:41397, datanodeUuid=5e400a8b-e68b-4fbe-8406-f707b7cad460, infoPort=38915, infoSecurePort=0, ipcPort=36023, storageInfo=lv=-57;cid=testClusterID;nsid=678129203;c=1732422620375)
2024-11-24T04:30:20,902 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x76243acccb837399 with lease ID 0xff875ab8a6d5170e: from storage DS-40d508c5-83fc-4808-89de-d8587e88042a node DatanodeRegistration(127.0.0.1:41397, datanodeUuid=5e400a8b-e68b-4fbe-8406-f707b7cad460, infoPort=38915, infoSecurePort=0, ipcPort=36023, storageInfo=lv=-57;cid=testClusterID;nsid=678129203;c=1732422620375), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T04:30:20,937 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7091f2a1{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f4065fa-2df8-4567-fbb0-481a3f88ce22/java.io.tmpdir/jetty-localhost-43823-hadoop-hdfs-3_4_1-tests_jar-_-any-8544639346857949378/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T04:30:20,937 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@249ce69c{HTTP/1.1, (http/1.1)}{localhost:43823}
2024-11-24T04:30:20,937 INFO  [Time-limited test {}] server.Server(415): Started @149999ms
2024-11-24T04:30:20,939 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-24T04:30:21,043 WARN  [Thread-1216 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f4065fa-2df8-4567-fbb0-481a3f88ce22/cluster_7730fbad-df8e-ea9e-97b3-261c7f308fa8/data/data3/current/BP-878497079-172.17.0.2-1732422620375/current, will proceed with Du for space computation calculation, 
2024-11-24T04:30:21,043 WARN  [Thread-1217 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f4065fa-2df8-4567-fbb0-481a3f88ce22/cluster_7730fbad-df8e-ea9e-97b3-261c7f308fa8/data/data4/current/BP-878497079-172.17.0.2-1732422620375/current, will proceed with Du for space computation calculation, 
2024-11-24T04:30:21,061 WARN  [Thread-1205 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-24T04:30:21,063 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7809c89a9a3e3403 with lease ID 0xff875ab8a6d5170f: Processing first storage report for DS-3b0df271-2496-41f3-863f-8757c97002a8 from datanode DatanodeRegistration(127.0.0.1:46273, datanodeUuid=8339f64a-a65e-4a7c-aeab-6bb7febc774a, infoPort=35957, infoSecurePort=0, ipcPort=46037, storageInfo=lv=-57;cid=testClusterID;nsid=678129203;c=1732422620375)
2024-11-24T04:30:21,063 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7809c89a9a3e3403 with lease ID 0xff875ab8a6d5170f: from storage DS-3b0df271-2496-41f3-863f-8757c97002a8 node DatanodeRegistration(127.0.0.1:46273, datanodeUuid=8339f64a-a65e-4a7c-aeab-6bb7febc774a, infoPort=35957, infoSecurePort=0, ipcPort=46037, storageInfo=lv=-57;cid=testClusterID;nsid=678129203;c=1732422620375), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T04:30:21,063 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7809c89a9a3e3403 with lease ID 0xff875ab8a6d5170f: Processing first storage report for DS-3eb47113-507d-4e64-99e0-a84100107583 from datanode DatanodeRegistration(127.0.0.1:46273, datanodeUuid=8339f64a-a65e-4a7c-aeab-6bb7febc774a, infoPort=35957, infoSecurePort=0, ipcPort=46037, storageInfo=lv=-57;cid=testClusterID;nsid=678129203;c=1732422620375)
2024-11-24T04:30:21,063 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7809c89a9a3e3403 with lease ID 0xff875ab8a6d5170f: from storage DS-3eb47113-507d-4e64-99e0-a84100107583 node DatanodeRegistration(127.0.0.1:46273, datanodeUuid=8339f64a-a65e-4a7c-aeab-6bb7febc774a, infoPort=35957, infoSecurePort=0, ipcPort=46037, storageInfo=lv=-57;cid=testClusterID;nsid=678129203;c=1732422620375), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T04:30:21,163 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f4065fa-2df8-4567-fbb0-481a3f88ce22
2024-11-24T04:30:21,166 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f4065fa-2df8-4567-fbb0-481a3f88ce22/cluster_7730fbad-df8e-ea9e-97b3-261c7f308fa8/zookeeper_0, clientPort=51577, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f4065fa-2df8-4567-fbb0-481a3f88ce22/cluster_7730fbad-df8e-ea9e-97b3-261c7f308fa8/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f4065fa-2df8-4567-fbb0-481a3f88ce22/cluster_7730fbad-df8e-ea9e-97b3-261c7f308fa8/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-24T04:30:21,167 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=51577
2024-11-24T04:30:21,167 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T04:30:21,168 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T04:30:21,177 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46273 is added to blk_1073741825_1001 (size=7)
2024-11-24T04:30:21,177 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41397 is added to blk_1073741825_1001 (size=7)
2024-11-24T04:30:21,178 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf with version=8
2024-11-24T04:30:21,178 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/hbase-staging
2024-11-24T04:30:21,180 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/fadb2fe0ba32:0 server-side Connection retries=45
2024-11-24T04:30:21,181 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-24T04:30:21,181 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-24T04:30:21,181 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-24T04:30:21,181 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-24T04:30:21,181 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-24T04:30:21,181 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-24T04:30:21,181 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-24T04:30:21,182 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39473
2024-11-24T04:30:21,183 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39473 connecting to ZooKeeper ensemble=127.0.0.1:51577
2024-11-24T04:30:21,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:394730x0, quorum=127.0.0.1:51577, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-24T04:30:21,196 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39473-0x101483d427d0000 connected
2024-11-24T04:30:21,215 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T04:30:21,217 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T04:30:21,219 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39473-0x101483d427d0000, quorum=127.0.0.1:51577, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-24T04:30:21,219 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf, hbase.cluster.distributed=false
2024-11-24T04:30:21,222 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39473-0x101483d427d0000, quorum=127.0.0.1:51577, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-24T04:30:21,222 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39473
2024-11-24T04:30:21,224 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39473
2024-11-24T04:30:21,227 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39473
2024-11-24T04:30:21,227 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39473
2024-11-24T04:30:21,228 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39473
2024-11-24T04:30:21,247 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/fadb2fe0ba32:0 server-side Connection retries=45
2024-11-24T04:30:21,247 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-24T04:30:21,247 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-24T04:30:21,247 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-24T04:30:21,247 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-24T04:30:21,247 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-24T04:30:21,247 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-24T04:30:21,247 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-24T04:30:21,248 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35619
2024-11-24T04:30:21,249 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35619 connecting to ZooKeeper ensemble=127.0.0.1:51577
2024-11-24T04:30:21,251 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T04:30:21,253 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T04:30:21,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:356190x0, quorum=127.0.0.1:51577, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-24T04:30:21,259 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:356190x0, quorum=127.0.0.1:51577, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-24T04:30:21,259 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35619-0x101483d427d0001 connected
2024-11-24T04:30:21,259 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-24T04:30:21,260 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-24T04:30:21,261 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35619-0x101483d427d0001, quorum=127.0.0.1:51577, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-24T04:30:21,261 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35619-0x101483d427d0001, quorum=127.0.0.1:51577, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-24T04:30:21,262 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35619
2024-11-24T04:30:21,262 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35619
2024-11-24T04:30:21,262 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35619
2024-11-24T04:30:21,263 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35619
2024-11-24T04:30:21,263 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35619
2024-11-24T04:30:21,277 DEBUG [M:0;fadb2fe0ba32:39473 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;fadb2fe0ba32:39473
2024-11-24T04:30:21,278 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/fadb2fe0ba32,39473,1732422621180
2024-11-24T04:30:21,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35619-0x101483d427d0001, quorum=127.0.0.1:51577, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-24T04:30:21,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39473-0x101483d427d0000, quorum=127.0.0.1:51577, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-24T04:30:21,280 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39473-0x101483d427d0000, quorum=127.0.0.1:51577, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/fadb2fe0ba32,39473,1732422621180
2024-11-24T04:30:21,282 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39473-0x101483d427d0000, quorum=127.0.0.1:51577, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:30:21,282 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35619-0x101483d427d0001, quorum=127.0.0.1:51577, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-24T04:30:21,282 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35619-0x101483d427d0001, quorum=127.0.0.1:51577, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:30:21,282 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39473-0x101483d427d0000, quorum=127.0.0.1:51577, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-24T04:30:21,283 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/fadb2fe0ba32,39473,1732422621180 from backup master directory
2024-11-24T04:30:21,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39473-0x101483d427d0000, quorum=127.0.0.1:51577, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/fadb2fe0ba32,39473,1732422621180
2024-11-24T04:30:21,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35619-0x101483d427d0001, quorum=127.0.0.1:51577, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-24T04:30:21,284 WARN  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-24T04:30:21,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39473-0x101483d427d0000, quorum=127.0.0.1:51577, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-24T04:30:21,284 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=fadb2fe0ba32,39473,1732422621180
2024-11-24T04:30:21,288 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/hbase.id] with ID: b4285d2a-51ef-4fba-bcd5-ae263ba011f1
2024-11-24T04:30:21,288 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/.tmp/hbase.id
2024-11-24T04:30:21,293 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46273 is added to blk_1073741826_1002 (size=42)
2024-11-24T04:30:21,294 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41397 is added to blk_1073741826_1002 (size=42)
2024-11-24T04:30:21,294 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/.tmp/hbase.id]:[hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/hbase.id]
2024-11-24T04:30:21,307 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T04:30:21,307 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-24T04:30:21,309 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms.
2024-11-24T04:30:21,312 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39473-0x101483d427d0000, quorum=127.0.0.1:51577, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:30:21,312 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35619-0x101483d427d0001, quorum=127.0.0.1:51577, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:30:21,318 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41397 is added to blk_1073741827_1003 (size=196)
2024-11-24T04:30:21,319 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46273 is added to blk_1073741827_1003 (size=196)
2024-11-24T04:30:21,319 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-24T04:30:21,320 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-24T04:30:21,320 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-24T04:30:21,328 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46273 is added to blk_1073741828_1004 (size=1189)
2024-11-24T04:30:21,329 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41397 is added to blk_1073741828_1004 (size=1189)
2024-11-24T04:30:21,330 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/MasterData/data/master/store
2024-11-24T04:30:21,336 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41397 is added to blk_1073741829_1005 (size=34)
2024-11-24T04:30:21,336 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46273 is added to blk_1073741829_1005 (size=34)
2024-11-24T04:30:21,336 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T04:30:21,337 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-24T04:30:21,337 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T04:30:21,337 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T04:30:21,337 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-24T04:30:21,337 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T04:30:21,337 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T04:30:21,337 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732422621337Disabling compacts and flushes for region at 1732422621337Disabling writes for close at 1732422621337Writing region close event to WAL at 1732422621337Closed at 1732422621337
2024-11-24T04:30:21,338 WARN  [master/fadb2fe0ba32:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/MasterData/data/master/store/.initializing
2024-11-24T04:30:21,338 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/MasterData/WALs/fadb2fe0ba32,39473,1732422621180
2024-11-24T04:30:21,341 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fadb2fe0ba32%2C39473%2C1732422621180, suffix=, logDir=hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/MasterData/WALs/fadb2fe0ba32,39473,1732422621180, archiveDir=hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/MasterData/oldWALs, maxLogs=10
2024-11-24T04:30:21,341 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor fadb2fe0ba32%2C39473%2C1732422621180.1732422621341
2024-11-24T04:30:21,347 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/MasterData/WALs/fadb2fe0ba32,39473,1732422621180/fadb2fe0ba32%2C39473%2C1732422621180.1732422621341
2024-11-24T04:30:21,348 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35957:35957),(127.0.0.1/127.0.0.1:38915:38915)]
2024-11-24T04:30:21,352 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-24T04:30:21,353 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T04:30:21,353 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T04:30:21,353 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T04:30:21,354 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-24T04:30:21,356 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-24T04:30:21,356 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:30:21,356 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T04:30:21,356 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-24T04:30:21,358 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-24T04:30:21,358 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:30:21,358 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-24T04:30:21,358 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-24T04:30:21,359 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-24T04:30:21,359 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:30:21,360 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-24T04:30:21,360 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-24T04:30:21,361 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-24T04:30:21,361 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:30:21,362 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-24T04:30:21,362 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T04:30:21,363 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-24T04:30:21,363 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-24T04:30:21,364 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T04:30:21,365 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T04:30:21,365 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-24T04:30:21,366 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T04:30:21,369 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-24T04:30:21,369 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=870655, jitterRate=0.10709568858146667}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-24T04:30:21,370 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1732422621353Initializing all the Stores at 1732422621354 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732422621354Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422621354Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422621354Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422621354Cleaning up temporary data from old regions at 1732422621365 (+11 ms)Region opened successfully at 1732422621370 (+5 ms)
2024-11-24T04:30:21,371 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-24T04:30:21,374 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e909dff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fadb2fe0ba32/172.17.0.2:0
2024-11-24T04:30:21,375 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-24T04:30:21,375 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-24T04:30:21,375 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-24T04:30:21,376 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-24T04:30:21,376 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-24T04:30:21,377 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-24T04:30:21,377 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-24T04:30:21,379 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-24T04:30:21,380 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39473-0x101483d427d0000, quorum=127.0.0.1:51577, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-24T04:30:21,381 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-24T04:30:21,382 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-24T04:30:21,382 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39473-0x101483d427d0000, quorum=127.0.0.1:51577, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-24T04:30:21,384 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-24T04:30:21,384 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-24T04:30:21,385 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39473-0x101483d427d0000, quorum=127.0.0.1:51577, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-24T04:30:21,386 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-24T04:30:21,388 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39473-0x101483d427d0000, quorum=127.0.0.1:51577, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-24T04:30:21,391 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-24T04:30:21,393 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39473-0x101483d427d0000, quorum=127.0.0.1:51577, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-24T04:30:21,394 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-24T04:30:21,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39473-0x101483d427d0000, quorum=127.0.0.1:51577, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-24T04:30:21,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35619-0x101483d427d0001, quorum=127.0.0.1:51577, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-24T04:30:21,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39473-0x101483d427d0000, quorum=127.0.0.1:51577, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:30:21,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35619-0x101483d427d0001, quorum=127.0.0.1:51577, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:30:21,396 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=fadb2fe0ba32,39473,1732422621180, sessionid=0x101483d427d0000, setting cluster-up flag (Was=false)
2024-11-24T04:30:21,400 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39473-0x101483d427d0000, quorum=127.0.0.1:51577, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:30:21,400 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35619-0x101483d427d0001, quorum=127.0.0.1:51577, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:30:21,405 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-24T04:30:21,406 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=fadb2fe0ba32,39473,1732422621180
2024-11-24T04:30:21,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39473-0x101483d427d0000, quorum=127.0.0.1:51577, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:30:21,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35619-0x101483d427d0001, quorum=127.0.0.1:51577, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:30:21,418 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-24T04:30:21,419 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=fadb2fe0ba32,39473,1732422621180
2024-11-24T04:30:21,420 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-24T04:30:21,422 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-24T04:30:21,422 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-24T04:30:21,422 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-24T04:30:21,423 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: fadb2fe0ba32,39473,1732422621180
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-24T04:30:21,424 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/fadb2fe0ba32:0, corePoolSize=5, maxPoolSize=5
2024-11-24T04:30:21,424 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/fadb2fe0ba32:0, corePoolSize=5, maxPoolSize=5
2024-11-24T04:30:21,424 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/fadb2fe0ba32:0, corePoolSize=5, maxPoolSize=5
2024-11-24T04:30:21,424 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/fadb2fe0ba32:0, corePoolSize=5, maxPoolSize=5
2024-11-24T04:30:21,424 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/fadb2fe0ba32:0, corePoolSize=10, maxPoolSize=10
2024-11-24T04:30:21,424 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:30:21,424 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/fadb2fe0ba32:0, corePoolSize=2, maxPoolSize=2
2024-11-24T04:30:21,424 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:30:21,425 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732422651425
2024-11-24T04:30:21,425 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-24T04:30:21,426 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-24T04:30:21,426 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-24T04:30:21,426 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-24T04:30:21,426 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-24T04:30:21,426 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-24T04:30:21,426 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-24T04:30:21,426 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-24T04:30:21,426 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-24T04:30:21,426 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-24T04:30:21,427 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-24T04:30:21,427 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-24T04:30:21,428 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:30:21,428 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-24T04:30:21,431 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-24T04:30:21,431 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-24T04:30:21,431 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/fadb2fe0ba32:0:becomeActiveMaster-HFileCleaner.large.0-1732422621431,5,FailOnTimeoutGroup]
2024-11-24T04:30:21,431 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/fadb2fe0ba32:0:becomeActiveMaster-HFileCleaner.small.0-1732422621431,5,FailOnTimeoutGroup]
2024-11-24T04:30:21,431 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-24T04:30:21,431 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-24T04:30:21,431 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-24T04:30:21,431 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-24T04:30:21,437 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46273 is added to blk_1073741831_1007 (size=1321)
2024-11-24T04:30:21,437 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41397 is added to blk_1073741831_1007 (size=1321)
2024-11-24T04:30:21,438 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-24T04:30:21,438 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf
2024-11-24T04:30:21,446 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41397 is added to blk_1073741832_1008 (size=32)
2024-11-24T04:30:21,446 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46273 is added to blk_1073741832_1008 (size=32)
2024-11-24T04:30:21,448 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T04:30:21,450 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-24T04:30:21,452 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-24T04:30:21,452 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:30:21,453 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T04:30:21,453 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-24T04:30:21,456 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-24T04:30:21,456 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:30:21,456 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T04:30:21,457 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-24T04:30:21,458 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-24T04:30:21,458 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:30:21,459 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T04:30:21,459 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-24T04:30:21,460 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-24T04:30:21,461 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:30:21,461 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T04:30:21,461 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-24T04:30:21,462 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/data/hbase/meta/1588230740
2024-11-24T04:30:21,462 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/data/hbase/meta/1588230740
2024-11-24T04:30:21,464 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-24T04:30:21,464 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-24T04:30:21,465 INFO  [RS:0;fadb2fe0ba32:35619 {}] regionserver.HRegionServer(746): ClusterId : b4285d2a-51ef-4fba-bcd5-ae263ba011f1
2024-11-24T04:30:21,465 DEBUG [RS:0;fadb2fe0ba32:35619 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-24T04:30:21,465 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-24T04:30:21,467 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-24T04:30:21,468 DEBUG [RS:0;fadb2fe0ba32:35619 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-24T04:30:21,469 DEBUG [RS:0;fadb2fe0ba32:35619 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-24T04:30:21,471 DEBUG [RS:0;fadb2fe0ba32:35619 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-24T04:30:21,471 DEBUG [RS:0;fadb2fe0ba32:35619 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38f38d47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fadb2fe0ba32/172.17.0.2:0
2024-11-24T04:30:21,473 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-24T04:30:21,474 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=855432, jitterRate=0.08773863315582275}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-24T04:30:21,475 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1732422621448Initializing all the Stores at 1732422621449 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732422621450 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732422621450Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422621450Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732422621450Cleaning up temporary data from old regions at 1732422621464 (+14 ms)Region opened successfully at 1732422621475 (+11 ms)
2024-11-24T04:30:21,475 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-24T04:30:21,475 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-24T04:30:21,475 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-24T04:30:21,475 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-24T04:30:21,475 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-24T04:30:21,481 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-24T04:30:21,481 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732422621475Disabling compacts and flushes for region at 1732422621475Disabling writes for close at 1732422621475Writing region close event to WAL at 1732422621481 (+6 ms)Closed at 1732422621481
2024-11-24T04:30:21,483 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-24T04:30:21,483 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-24T04:30:21,483 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-24T04:30:21,485 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-24T04:30:21,488 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-24T04:30:21,491 DEBUG [RS:0;fadb2fe0ba32:35619 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;fadb2fe0ba32:35619
2024-11-24T04:30:21,491 INFO  [RS:0;fadb2fe0ba32:35619 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-24T04:30:21,491 INFO  [RS:0;fadb2fe0ba32:35619 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-24T04:30:21,491 DEBUG [RS:0;fadb2fe0ba32:35619 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-24T04:30:21,492 INFO  [RS:0;fadb2fe0ba32:35619 {}] regionserver.HRegionServer(2659): reportForDuty to master=fadb2fe0ba32,39473,1732422621180 with port=35619, startcode=1732422621246
2024-11-24T04:30:21,493 DEBUG [RS:0;fadb2fe0ba32:35619 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-24T04:30:21,495 INFO  [HMaster-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43151, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.4 (auth:SIMPLE), service=RegionServerStatusService
2024-11-24T04:30:21,496 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39473 {}] master.ServerManager(363): Checking decommissioned status of RegionServer fadb2fe0ba32,35619,1732422621246
2024-11-24T04:30:21,496 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39473 {}] master.ServerManager(517): Registering regionserver=fadb2fe0ba32,35619,1732422621246
2024-11-24T04:30:21,498 DEBUG [RS:0;fadb2fe0ba32:35619 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf
2024-11-24T04:30:21,498 DEBUG [RS:0;fadb2fe0ba32:35619 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38691
2024-11-24T04:30:21,498 DEBUG [RS:0;fadb2fe0ba32:35619 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-24T04:30:21,499 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39473-0x101483d427d0000, quorum=127.0.0.1:51577, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-24T04:30:21,500 DEBUG [RS:0;fadb2fe0ba32:35619 {}] zookeeper.ZKUtil(111): regionserver:35619-0x101483d427d0001, quorum=127.0.0.1:51577, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/fadb2fe0ba32,35619,1732422621246
2024-11-24T04:30:21,500 WARN  [RS:0;fadb2fe0ba32:35619 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-24T04:30:21,500 INFO  [RS:0;fadb2fe0ba32:35619 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-24T04:30:21,500 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [fadb2fe0ba32,35619,1732422621246]
2024-11-24T04:30:21,500 DEBUG [RS:0;fadb2fe0ba32:35619 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246
2024-11-24T04:30:21,504 INFO  [RS:0;fadb2fe0ba32:35619 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-24T04:30:21,510 INFO  [RS:0;fadb2fe0ba32:35619 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-24T04:30:21,510 INFO  [RS:0;fadb2fe0ba32:35619 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-24T04:30:21,510 INFO  [RS:0;fadb2fe0ba32:35619 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-24T04:30:21,510 INFO  [RS:0;fadb2fe0ba32:35619 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-24T04:30:21,511 INFO  [RS:0;fadb2fe0ba32:35619 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-24T04:30:21,511 INFO  [RS:0;fadb2fe0ba32:35619 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-24T04:30:21,512 DEBUG [RS:0;fadb2fe0ba32:35619 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:30:21,512 DEBUG [RS:0;fadb2fe0ba32:35619 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:30:21,512 DEBUG [RS:0;fadb2fe0ba32:35619 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:30:21,512 DEBUG [RS:0;fadb2fe0ba32:35619 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:30:21,512 DEBUG [RS:0;fadb2fe0ba32:35619 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:30:21,512 DEBUG [RS:0;fadb2fe0ba32:35619 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/fadb2fe0ba32:0, corePoolSize=2, maxPoolSize=2
2024-11-24T04:30:21,512 DEBUG [RS:0;fadb2fe0ba32:35619 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:30:21,512 DEBUG [RS:0;fadb2fe0ba32:35619 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:30:21,512 DEBUG [RS:0;fadb2fe0ba32:35619 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:30:21,512 DEBUG [RS:0;fadb2fe0ba32:35619 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:30:21,512 DEBUG [RS:0;fadb2fe0ba32:35619 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:30:21,512 DEBUG [RS:0;fadb2fe0ba32:35619 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:30:21,512 DEBUG [RS:0;fadb2fe0ba32:35619 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/fadb2fe0ba32:0, corePoolSize=3, maxPoolSize=3
2024-11-24T04:30:21,512 DEBUG [RS:0;fadb2fe0ba32:35619 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/fadb2fe0ba32:0, corePoolSize=3, maxPoolSize=3
2024-11-24T04:30:21,512 INFO  [RS:0;fadb2fe0ba32:35619 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-24T04:30:21,512 INFO  [RS:0;fadb2fe0ba32:35619 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-24T04:30:21,513 INFO  [RS:0;fadb2fe0ba32:35619 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-24T04:30:21,513 INFO  [RS:0;fadb2fe0ba32:35619 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-24T04:30:21,513 INFO  [RS:0;fadb2fe0ba32:35619 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-24T04:30:21,513 INFO  [RS:0;fadb2fe0ba32:35619 {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,35619,1732422621246-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-24T04:30:21,528 INFO  [RS:0;fadb2fe0ba32:35619 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-24T04:30:21,529 INFO  [RS:0;fadb2fe0ba32:35619 {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,35619,1732422621246-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-24T04:30:21,529 INFO  [RS:0;fadb2fe0ba32:35619 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T04:30:21,529 INFO  [RS:0;fadb2fe0ba32:35619 {}] regionserver.Replication(171): fadb2fe0ba32,35619,1732422621246 started
2024-11-24T04:30:21,544 INFO  [RS:0;fadb2fe0ba32:35619 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T04:30:21,544 INFO  [RS:0;fadb2fe0ba32:35619 {}] regionserver.HRegionServer(1482): Serving as fadb2fe0ba32,35619,1732422621246, RpcServer on fadb2fe0ba32/172.17.0.2:35619, sessionid=0x101483d427d0001
2024-11-24T04:30:21,544 DEBUG [RS:0;fadb2fe0ba32:35619 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-24T04:30:21,544 DEBUG [RS:0;fadb2fe0ba32:35619 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager fadb2fe0ba32,35619,1732422621246
2024-11-24T04:30:21,544 DEBUG [RS:0;fadb2fe0ba32:35619 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fadb2fe0ba32,35619,1732422621246'
2024-11-24T04:30:21,544 DEBUG [RS:0;fadb2fe0ba32:35619 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-24T04:30:21,545 DEBUG [RS:0;fadb2fe0ba32:35619 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-24T04:30:21,545 DEBUG [RS:0;fadb2fe0ba32:35619 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-24T04:30:21,545 DEBUG [RS:0;fadb2fe0ba32:35619 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-24T04:30:21,545 DEBUG [RS:0;fadb2fe0ba32:35619 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager fadb2fe0ba32,35619,1732422621246
2024-11-24T04:30:21,545 DEBUG [RS:0;fadb2fe0ba32:35619 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fadb2fe0ba32,35619,1732422621246'
2024-11-24T04:30:21,545 DEBUG [RS:0;fadb2fe0ba32:35619 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-24T04:30:21,546 DEBUG [RS:0;fadb2fe0ba32:35619 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-24T04:30:21,546 DEBUG [RS:0;fadb2fe0ba32:35619 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-24T04:30:21,546 INFO  [RS:0;fadb2fe0ba32:35619 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-24T04:30:21,546 INFO  [RS:0;fadb2fe0ba32:35619 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-24T04:30:21,639 WARN  [fadb2fe0ba32:39473 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-11-24T04:30:21,648 INFO  [RS:0;fadb2fe0ba32:35619 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fadb2fe0ba32%2C35619%2C1732422621246, suffix=, logDir=hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246, archiveDir=hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/oldWALs, maxLogs=32
2024-11-24T04:30:21,649 INFO  [RS:0;fadb2fe0ba32:35619 {}] monitor.StreamSlowMonitor(122): New stream slow monitor fadb2fe0ba32%2C35619%2C1732422621246.1732422621649
2024-11-24T04:30:21,655 INFO  [RS:0;fadb2fe0ba32:35619 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.1732422621649
2024-11-24T04:30:21,656 DEBUG [RS:0;fadb2fe0ba32:35619 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35957:35957),(127.0.0.1/127.0.0.1:38915:38915)]
2024-11-24T04:30:21,759 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:21,799 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:21,889 DEBUG [fadb2fe0ba32:39473 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-24T04:30:21,890 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=fadb2fe0ba32,35619,1732422621246
2024-11-24T04:30:21,891 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fadb2fe0ba32,35619,1732422621246, state=OPENING
2024-11-24T04:30:21,893 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-24T04:30:21,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39473-0x101483d427d0000, quorum=127.0.0.1:51577, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:30:21,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35619-0x101483d427d0001, quorum=127.0.0.1:51577, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:30:21,895 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-24T04:30:21,895 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=fadb2fe0ba32,35619,1732422621246}]
2024-11-24T04:30:21,895 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-24T04:30:21,895 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-24T04:30:22,048 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-24T04:30:22,050 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39385, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-24T04:30:22,054 INFO  [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-24T04:30:22,054 INFO  [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-24T04:30:22,057 INFO  [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fadb2fe0ba32%2C35619%2C1732422621246.meta, suffix=.meta, logDir=hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246, archiveDir=hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/oldWALs, maxLogs=32
2024-11-24T04:30:22,058 INFO  [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor fadb2fe0ba32%2C35619%2C1732422621246.meta.1732422622057.meta
2024-11-24T04:30:22,065 INFO  [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.meta.1732422622057.meta
2024-11-24T04:30:22,066 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38915:38915),(127.0.0.1/127.0.0.1:35957:35957)]
2024-11-24T04:30:22,067 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-24T04:30:22,067 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-24T04:30:22,068 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-24T04:30:22,068 INFO  [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-24T04:30:22,068 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-24T04:30:22,068 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T04:30:22,068 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-24T04:30:22,068 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-24T04:30:22,071 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-24T04:30:22,072 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-24T04:30:22,072 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:30:22,073 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T04:30:22,073 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-24T04:30:22,074 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-24T04:30:22,074 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:30:22,075 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T04:30:22,075 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-24T04:30:22,076 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-24T04:30:22,076 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:30:22,077 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T04:30:22,077 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-24T04:30:22,078 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-24T04:30:22,078 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:30:22,078 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T04:30:22,078 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-24T04:30:22,079 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/data/hbase/meta/1588230740
2024-11-24T04:30:22,080 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/data/hbase/meta/1588230740
2024-11-24T04:30:22,081 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-24T04:30:22,081 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-24T04:30:22,082 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-24T04:30:22,083 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-24T04:30:22,084 INFO  [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=732154, jitterRate=-0.06901895999908447}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-24T04:30:22,084 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-24T04:30:22,085 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1732422622068Writing region info on filesystem at 1732422622068Initializing all the Stores at 1732422622069 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732422622069Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732422622071 (+2 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422622071Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732422622071Cleaning up temporary data from old regions at 1732422622081 (+10 ms)Running coprocessor post-open hooks at 1732422622084 (+3 ms)Region opened successfully at 1732422622085 (+1 ms)
2024-11-24T04:30:22,086 INFO  [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732422622048
2024-11-24T04:30:22,088 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-24T04:30:22,088 INFO  [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-24T04:30:22,089 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=fadb2fe0ba32,35619,1732422621246
2024-11-24T04:30:22,090 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fadb2fe0ba32,35619,1732422621246, state=OPEN
2024-11-24T04:30:22,095 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35619-0x101483d427d0001, quorum=127.0.0.1:51577, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-24T04:30:22,095 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39473-0x101483d427d0000, quorum=127.0.0.1:51577, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-24T04:30:22,095 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=fadb2fe0ba32,35619,1732422621246
2024-11-24T04:30:22,095 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-24T04:30:22,095 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-24T04:30:22,099 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-24T04:30:22,100 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fadb2fe0ba32,35619,1732422621246 in 200 msec
2024-11-24T04:30:22,102 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-24T04:30:22,103 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 617 msec
2024-11-24T04:30:22,104 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-24T04:30:22,104 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-24T04:30:22,105 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-24T04:30:22,106 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fadb2fe0ba32,35619,1732422621246, seqNum=-1]
2024-11-24T04:30:22,106 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-24T04:30:22,107 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-11-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57201, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-24T04:30:22,113 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 691 msec
2024-11-24T04:30:22,113 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732422622113, completionTime=-1
2024-11-24T04:30:22,114 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-24T04:30:22,114 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-24T04:30:22,115 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-24T04:30:22,115 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732422682115
2024-11-24T04:30:22,116 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732422742116
2024-11-24T04:30:22,116 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 1 msec
2024-11-24T04:30:22,116 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,39473,1732422621180-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-24T04:30:22,116 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,39473,1732422621180-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T04:30:22,116 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,39473,1732422621180-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T04:30:22,116 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-fadb2fe0ba32:39473, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T04:30:22,116 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-24T04:30:22,116 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-24T04:30:22,118 DEBUG [master/fadb2fe0ba32:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-24T04:30:22,120 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.836sec
2024-11-24T04:30:22,120 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-24T04:30:22,120 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-24T04:30:22,120 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-24T04:30:22,120 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-24T04:30:22,120 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-24T04:30:22,120 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,39473,1732422621180-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-24T04:30:22,120 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,39473,1732422621180-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-24T04:30:22,123 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-24T04:30:22,123 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-24T04:30:22,123 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,39473,1732422621180-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T04:30:22,165 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@220d6e9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-24T04:30:22,165 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fadb2fe0ba32,39473,-1 for getting cluster id
2024-11-24T04:30:22,166 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-24T04:30:22,167 DEBUG [HMaster-EventLoopGroup-10-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b4285d2a-51ef-4fba-bcd5-ae263ba011f1'
2024-11-24T04:30:22,168 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-24T04:30:22,168 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b4285d2a-51ef-4fba-bcd5-ae263ba011f1"

2024-11-24T04:30:22,168 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1af89d9a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-24T04:30:22,168 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fadb2fe0ba32,39473,-1]
2024-11-24T04:30:22,169 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-24T04:30:22,169 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T04:30:22,170 INFO  [HMaster-EventLoopGroup-10-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48216, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-24T04:30:22,171 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@752fee05, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-24T04:30:22,172 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-24T04:30:22,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fadb2fe0ba32,35619,1732422621246, seqNum=-1]
2024-11-24T04:30:22,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-24T04:30:22,174 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47464, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-24T04:30:22,176 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=fadb2fe0ba32,39473,1732422621180
2024-11-24T04:30:22,176 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T04:30:22,179 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-24T04:30:22,179 INFO  [Time-limited test {}] wal.TestLogRolling(320): Starting testLogRollOnPipelineRestart
2024-11-24T04:30:22,179 INFO  [Time-limited test {}] wal.TestLogRolling(323): Replication=2
2024-11-24T04:30:22,179 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-24T04:30:22,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncConnectionImpl(321): The fetched master address is fadb2fe0ba32,39473,1732422621180
2024-11-24T04:30:22,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2a2efa67
2024-11-24T04:30:22,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-24T04:30:22,182 INFO  [HMaster-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48230, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-24T04:30:22,182 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39473 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-24T04:30:22,182 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39473 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-24T04:30:22,183 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39473 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRollOnPipelineRestart', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-24T04:30:22,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39473 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart
2024-11-24T04:30:22,185 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_PRE_OPERATION
2024-11-24T04:30:22,185 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:30:22,185 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39473 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRollOnPipelineRestart"
 procId is: 4
2024-11-24T04:30:22,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39473 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-24T04:30:22,187 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-24T04:30:22,193 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41397 is added to blk_1073741835_1011 (size=395)
2024-11-24T04:30:22,194 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46273 is added to blk_1073741835_1011 (size=395)
2024-11-24T04:30:22,196 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8f02b50dcfd7865c201db72b9bcecab1, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1732422622182.8f02b50dcfd7865c201db72b9bcecab1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnPipelineRestart', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf
2024-11-24T04:30:22,206 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41397 is added to blk_1073741836_1012 (size=78)
2024-11-24T04:30:22,207 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46273 is added to blk_1073741836_1012 (size=78)
2024-11-24T04:30:22,208 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1732422622182.8f02b50dcfd7865c201db72b9bcecab1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T04:30:22,208 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1722): Closing 8f02b50dcfd7865c201db72b9bcecab1, disabling compactions & flushes
2024-11-24T04:30:22,208 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1732422622182.8f02b50dcfd7865c201db72b9bcecab1.
2024-11-24T04:30:22,208 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1732422622182.8f02b50dcfd7865c201db72b9bcecab1.
2024-11-24T04:30:22,208 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1732422622182.8f02b50dcfd7865c201db72b9bcecab1. after waiting 0 ms
2024-11-24T04:30:22,208 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1732422622182.8f02b50dcfd7865c201db72b9bcecab1.
2024-11-24T04:30:22,208 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnPipelineRestart,,1732422622182.8f02b50dcfd7865c201db72b9bcecab1.
2024-11-24T04:30:22,208 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8f02b50dcfd7865c201db72b9bcecab1:
Waiting for close lock at 1732422622208Disabling compacts and flushes for region at 1732422622208Disabling writes for close at 1732422622208Writing region close event to WAL at 1732422622208Closed at 1732422622208
2024-11-24T04:30:22,210 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ADD_TO_META
2024-11-24T04:30:22,210 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnPipelineRestart,,1732422622182.8f02b50dcfd7865c201db72b9bcecab1.","families":{"info":[{"qualifier":"regioninfo","vlen":77,"tag":[],"timestamp":"1732422622210"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732422622210"}]},"ts":"1732422622210"}
2024-11-24T04:30:22,213 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-24T04:30:22,215 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-24T04:30:22,215 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732422622215"}]},"ts":"1732422622215"}
2024-11-24T04:30:22,218 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLING in hbase:meta
2024-11-24T04:30:22,218 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=8f02b50dcfd7865c201db72b9bcecab1, ASSIGN}]
2024-11-24T04:30:22,220 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=8f02b50dcfd7865c201db72b9bcecab1, ASSIGN
2024-11-24T04:30:22,221 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=8f02b50dcfd7865c201db72b9bcecab1, ASSIGN; state=OFFLINE, location=fadb2fe0ba32,35619,1732422621246; forceNewPlan=false, retain=false
2024-11-24T04:30:22,372 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=8f02b50dcfd7865c201db72b9bcecab1, regionState=OPENING, regionLocation=fadb2fe0ba32,35619,1732422621246
2024-11-24T04:30:22,374 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-11-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=8f02b50dcfd7865c201db72b9bcecab1, ASSIGN because future has completed
2024-11-24T04:30:22,375 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8f02b50dcfd7865c201db72b9bcecab1, server=fadb2fe0ba32,35619,1732422621246}]
2024-11-24T04:30:22,532 INFO  [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRollOnPipelineRestart,,1732422622182.8f02b50dcfd7865c201db72b9bcecab1.
2024-11-24T04:30:22,532 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 8f02b50dcfd7865c201db72b9bcecab1, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1732422622182.8f02b50dcfd7865c201db72b9bcecab1.', STARTKEY => '', ENDKEY => ''}
2024-11-24T04:30:22,533 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnPipelineRestart 8f02b50dcfd7865c201db72b9bcecab1
2024-11-24T04:30:22,533 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1732422622182.8f02b50dcfd7865c201db72b9bcecab1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T04:30:22,533 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 8f02b50dcfd7865c201db72b9bcecab1
2024-11-24T04:30:22,533 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 8f02b50dcfd7865c201db72b9bcecab1
2024-11-24T04:30:22,534 INFO  [StoreOpener-8f02b50dcfd7865c201db72b9bcecab1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 8f02b50dcfd7865c201db72b9bcecab1 
2024-11-24T04:30:22,535 INFO  [StoreOpener-8f02b50dcfd7865c201db72b9bcecab1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f02b50dcfd7865c201db72b9bcecab1 columnFamilyName info
2024-11-24T04:30:22,536 DEBUG [StoreOpener-8f02b50dcfd7865c201db72b9bcecab1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:30:22,536 INFO  [StoreOpener-8f02b50dcfd7865c201db72b9bcecab1-1 {}] regionserver.HStore(327): Store=8f02b50dcfd7865c201db72b9bcecab1/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-24T04:30:22,536 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 8f02b50dcfd7865c201db72b9bcecab1
2024-11-24T04:30:22,537 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/data/default/TestLogRolling-testLogRollOnPipelineRestart/8f02b50dcfd7865c201db72b9bcecab1
2024-11-24T04:30:22,537 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/data/default/TestLogRolling-testLogRollOnPipelineRestart/8f02b50dcfd7865c201db72b9bcecab1
2024-11-24T04:30:22,538 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 8f02b50dcfd7865c201db72b9bcecab1
2024-11-24T04:30:22,538 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 8f02b50dcfd7865c201db72b9bcecab1
2024-11-24T04:30:22,539 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 8f02b50dcfd7865c201db72b9bcecab1
2024-11-24T04:30:22,542 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/data/default/TestLogRolling-testLogRollOnPipelineRestart/8f02b50dcfd7865c201db72b9bcecab1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-24T04:30:22,542 INFO  [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 8f02b50dcfd7865c201db72b9bcecab1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=738976, jitterRate=-0.06034421920776367}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-24T04:30:22,542 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8f02b50dcfd7865c201db72b9bcecab1
2024-11-24T04:30:22,543 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 8f02b50dcfd7865c201db72b9bcecab1:
Running coprocessor pre-open hook at 1732422622533Writing region info on filesystem at 1732422622533Initializing all the Stores at 1732422622534 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422622534Cleaning up temporary data from old regions at 1732422622538 (+4 ms)Running coprocessor post-open hooks at 1732422622542 (+4 ms)Region opened successfully at 1732422622543 (+1 ms)
2024-11-24T04:30:22,544 INFO  [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRollOnPipelineRestart,,1732422622182.8f02b50dcfd7865c201db72b9bcecab1., pid=6, masterSystemTime=1732422622528
2024-11-24T04:30:22,547 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRollOnPipelineRestart,,1732422622182.8f02b50dcfd7865c201db72b9bcecab1.
2024-11-24T04:30:22,547 INFO  [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRollOnPipelineRestart,,1732422622182.8f02b50dcfd7865c201db72b9bcecab1.
2024-11-24T04:30:22,548 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=8f02b50dcfd7865c201db72b9bcecab1, regionState=OPEN, openSeqNum=2, regionLocation=fadb2fe0ba32,35619,1732422621246
2024-11-24T04:30:22,550 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-11-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8f02b50dcfd7865c201db72b9bcecab1, server=fadb2fe0ba32,35619,1732422621246 because future has completed
2024-11-24T04:30:22,554 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-24T04:30:22,554 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 8f02b50dcfd7865c201db72b9bcecab1, server=fadb2fe0ba32,35619,1732422621246 in 177 msec
2024-11-24T04:30:22,557 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-24T04:30:22,557 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=8f02b50dcfd7865c201db72b9bcecab1, ASSIGN in 336 msec
2024-11-24T04:30:22,558 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-24T04:30:22,559 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732422622558"}]},"ts":"1732422622558"}
2024-11-24T04:30:22,561 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLED in hbase:meta
2024-11-24T04:30:22,562 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_POST_OPERATION
2024-11-24T04:30:22,565 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart in 379 msec
2024-11-24T04:30:22,759 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:22,800 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:23,760 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:23,801 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:24,761 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:24,801 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:25,761 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:25,802 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:26,266 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-24T04:30:26,266 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-11-24T04:30:26,267 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart
2024-11-24T04:30:26,267 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart Metrics about Tables on a single HBase RegionServer
2024-11-24T04:30:26,267 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-24T04:30:26,267 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-11-24T04:30:26,762 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:26,803 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:27,570 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-24T04:30:27,584 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:30:27,584 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:30:27,584 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:30:27,585 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:30:27,585 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:30:27,585 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:30:27,588 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:30:27,589 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:30:27,589 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:30:27,591 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:30:27,596 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-24T04:30:27,596 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnPipelineRestart'
2024-11-24T04:30:27,763 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:27,803 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:28,763 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:28,804 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:29,764 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:29,805 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:30,765 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:30,805 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:31,765 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:31,806 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:32,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39473 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-24T04:30:32,213 INFO  [RPCClient-NioEventLoopGroup-4-15 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnPipelineRestart completed
2024-11-24T04:30:32,213 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRollOnPipelineRestart,, stopping at row=TestLogRolling-testLogRollOnPipelineRestart ,, for max=2147483647 with caching=100
2024-11-24T04:30:32,216 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRollOnPipelineRestart
2024-11-24T04:30:32,216 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRollOnPipelineRestart,,1732422622182.8f02b50dcfd7865c201db72b9bcecab1.
2024-11-24T04:30:32,219 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRollOnPipelineRestart', row='row1002', locateType=CURRENT is [region=TestLogRolling-testLogRollOnPipelineRestart,,1732422622182.8f02b50dcfd7865c201db72b9bcecab1., hostname=fadb2fe0ba32,35619,1732422621246, seqNum=2]
2024-11-24T04:30:32,766 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:32,806 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:33,766 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:33,807 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:34,222 INFO  [Time-limited test {}] wal.TestLogRolling(360): log.getCurrentFileName()): hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.1732422621649
2024-11-24T04:30:34,223 WARN  [ResponseProcessor for block BP-878497079-172.17.0.2-1732422620375:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-878497079-172.17.0.2-1732422620375:blk_1073741833_1009
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:30:34,223 WARN  [ResponseProcessor for block BP-878497079-172.17.0.2-1732422620375:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-878497079-172.17.0.2-1732422620375:blk_1073741830_1006
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:30:34,223 WARN  [ResponseProcessor for block BP-878497079-172.17.0.2-1732422620375:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-878497079-172.17.0.2-1732422620375:blk_1073741834_1010
java.io.IOException: Bad response ERROR for BP-878497079-172.17.0.2-1732422620375:blk_1073741834_1010 from datanode DatanodeInfoWithStorage[127.0.0.1:46273,DS-3b0df271-2496-41f3-863f-8757c97002a8,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:30:34,223 WARN  [DataStreamer for file /user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/MasterData/WALs/fadb2fe0ba32,39473,1732422621180/fadb2fe0ba32%2C39473%2C1732422621180.1732422621341 block BP-878497079-172.17.0.2-1732422620375:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-878497079-172.17.0.2-1732422620375:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46273,DS-3b0df271-2496-41f3-863f-8757c97002a8,DISK], DatanodeInfoWithStorage[127.0.0.1:41397,DS-38ec15dd-c607-46cb-8f20-43e17ecaf8ee,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:46273,DS-3b0df271-2496-41f3-863f-8757c97002a8,DISK]) is bad.
2024-11-24T04:30:34,223 WARN  [DataStreamer for file /user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.1732422621649 block BP-878497079-172.17.0.2-1732422620375:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-878497079-172.17.0.2-1732422620375:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46273,DS-3b0df271-2496-41f3-863f-8757c97002a8,DISK], DatanodeInfoWithStorage[127.0.0.1:41397,DS-38ec15dd-c607-46cb-8f20-43e17ecaf8ee,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:46273,DS-3b0df271-2496-41f3-863f-8757c97002a8,DISK]) is bad.
2024-11-24T04:30:34,224 WARN  [DataStreamer for file /user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.meta.1732422622057.meta block BP-878497079-172.17.0.2-1732422620375:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-878497079-172.17.0.2-1732422620375:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41397,DS-38ec15dd-c607-46cb-8f20-43e17ecaf8ee,DISK], DatanodeInfoWithStorage[127.0.0.1:46273,DS-3b0df271-2496-41f3-863f-8757c97002a8,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:46273,DS-3b0df271-2496-41f3-863f-8757c97002a8,DISK]) is bad.
2024-11-24T04:30:34,224 WARN  [PacketResponder: BP-878497079-172.17.0.2-1732422620375:blk_1073741834_1010, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:46273] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T04:30:34,224 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_15500758_22 at /127.0.0.1:54016 [Receiving block BP-878497079-172.17.0.2-1732422620375:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:46273:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:54016 dst: /127.0.0.1:46273
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T04:30:34,225 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_15500758_22 at /127.0.0.1:47170 [Receiving block BP-878497079-172.17.0.2-1732422620375:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:41397:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:47170 dst: /127.0.0.1:41397
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T04:30:34,225 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-329496266_22 at /127.0.0.1:47138 [Receiving block BP-878497079-172.17.0.2-1732422620375:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:41397:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:47138 dst: /127.0.0.1:41397
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T04:30:34,224 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-329496266_22 at /127.0.0.1:53982 [Receiving block BP-878497079-172.17.0.2-1732422620375:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:46273:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:53982 dst: /127.0.0.1:46273
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T04:30:34,225 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_15500758_22 at /127.0.0.1:47178 [Receiving block BP-878497079-172.17.0.2-1732422620375:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:41397:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:47178 dst: /127.0.0.1:41397
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T04:30:34,225 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_15500758_22 at /127.0.0.1:54020 [Receiving block BP-878497079-172.17.0.2-1732422620375:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:46273:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:54020 dst: /127.0.0.1:46273
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T04:30:34,226 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7091f2a1{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T04:30:34,227 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@249ce69c{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-24T04:30:34,227 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-24T04:30:34,227 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@671953b5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-24T04:30:34,227 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7ec5b9af{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f4065fa-2df8-4567-fbb0-481a3f88ce22/hadoop.log.dir/,STOPPED}
2024-11-24T04:30:34,228 WARN  [BP-878497079-172.17.0.2-1732422620375 heartbeating to localhost/127.0.0.1:38691 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-24T04:30:34,228 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-24T04:30:34,228 WARN  [BP-878497079-172.17.0.2-1732422620375 heartbeating to localhost/127.0.0.1:38691 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-878497079-172.17.0.2-1732422620375 (Datanode Uuid 8339f64a-a65e-4a7c-aeab-6bb7febc774a) service to localhost/127.0.0.1:38691
2024-11-24T04:30:34,228 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-24T04:30:34,229 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f4065fa-2df8-4567-fbb0-481a3f88ce22/cluster_7730fbad-df8e-ea9e-97b3-261c7f308fa8/data/data3/current/BP-878497079-172.17.0.2-1732422620375 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T04:30:34,229 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f4065fa-2df8-4567-fbb0-481a3f88ce22/cluster_7730fbad-df8e-ea9e-97b3-261c7f308fa8/data/data4/current/BP-878497079-172.17.0.2-1732422620375 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T04:30:34,229 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-24T04:30:34,238 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T04:30:34,242 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-24T04:30:34,242 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-24T04:30:34,242 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-24T04:30:34,242 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-24T04:30:34,243 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@36974255{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f4065fa-2df8-4567-fbb0-481a3f88ce22/hadoop.log.dir/,AVAILABLE}
2024-11-24T04:30:34,243 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@126bd190{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-24T04:30:34,361 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@55cf8f7d{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f4065fa-2df8-4567-fbb0-481a3f88ce22/java.io.tmpdir/jetty-localhost-35719-hadoop-hdfs-3_4_1-tests_jar-_-any-12207947605402872047/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T04:30:34,362 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3de5676f{HTTP/1.1, (http/1.1)}{localhost:35719}
2024-11-24T04:30:34,362 INFO  [Time-limited test {}] server.Server(415): Started @163423ms
2024-11-24T04:30:34,364 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-24T04:30:34,389 WARN  [ResponseProcessor for block BP-878497079-172.17.0.2-1732422620375:blk_1073741833_1014 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-878497079-172.17.0.2-1732422620375:blk_1073741833_1014
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:30:34,389 WARN  [ResponseProcessor for block BP-878497079-172.17.0.2-1732422620375:blk_1073741830_1015 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-878497079-172.17.0.2-1732422620375:blk_1073741830_1015
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:30:34,389 WARN  [ResponseProcessor for block BP-878497079-172.17.0.2-1732422620375:blk_1073741834_1013 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-878497079-172.17.0.2-1732422620375:blk_1073741834_1013
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:30:34,390 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-329496266_22 at /127.0.0.1:36052 [Receiving block BP-878497079-172.17.0.2-1732422620375:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:41397:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:36052 dst: /127.0.0.1:41397
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T04:30:34,390 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_15500758_22 at /127.0.0.1:36048 [Receiving block BP-878497079-172.17.0.2-1732422620375:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:41397:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:36048 dst: /127.0.0.1:41397
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T04:30:34,390 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_15500758_22 at /127.0.0.1:36032 [Receiving block BP-878497079-172.17.0.2-1732422620375:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:41397:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:36032 dst: /127.0.0.1:41397
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T04:30:34,397 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5493c38f{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T04:30:34,397 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@50743f5b{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-24T04:30:34,397 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-24T04:30:34,397 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2744dc92{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-24T04:30:34,397 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2b6caabd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f4065fa-2df8-4567-fbb0-481a3f88ce22/hadoop.log.dir/,STOPPED}
2024-11-24T04:30:34,399 WARN  [BP-878497079-172.17.0.2-1732422620375 heartbeating to localhost/127.0.0.1:38691 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-24T04:30:34,399 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-24T04:30:34,399 WARN  [BP-878497079-172.17.0.2-1732422620375 heartbeating to localhost/127.0.0.1:38691 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-878497079-172.17.0.2-1732422620375 (Datanode Uuid 5e400a8b-e68b-4fbe-8406-f707b7cad460) service to localhost/127.0.0.1:38691
2024-11-24T04:30:34,399 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-24T04:30:34,401 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f4065fa-2df8-4567-fbb0-481a3f88ce22/cluster_7730fbad-df8e-ea9e-97b3-261c7f308fa8/data/data1/current/BP-878497079-172.17.0.2-1732422620375 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T04:30:34,401 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f4065fa-2df8-4567-fbb0-481a3f88ce22/cluster_7730fbad-df8e-ea9e-97b3-261c7f308fa8/data/data2/current/BP-878497079-172.17.0.2-1732422620375 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T04:30:34,402 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-24T04:30:34,410 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T04:30:34,414 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-24T04:30:34,414 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-24T04:30:34,414 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-24T04:30:34,415 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-24T04:30:34,415 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@43a7cd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f4065fa-2df8-4567-fbb0-481a3f88ce22/hadoop.log.dir/,AVAILABLE}
2024-11-24T04:30:34,415 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2e49578b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-24T04:30:34,468 WARN  [Thread-1340 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-24T04:30:34,471 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfe68d79266e8ac2a with lease ID 0xff875ab8a6d51710: from storage DS-3b0df271-2496-41f3-863f-8757c97002a8 node DatanodeRegistration(127.0.0.1:41225, datanodeUuid=8339f64a-a65e-4a7c-aeab-6bb7febc774a, infoPort=34371, infoSecurePort=0, ipcPort=39577, storageInfo=lv=-57;cid=testClusterID;nsid=678129203;c=1732422620375), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T04:30:34,471 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfe68d79266e8ac2a with lease ID 0xff875ab8a6d51710: from storage DS-3eb47113-507d-4e64-99e0-a84100107583 node DatanodeRegistration(127.0.0.1:41225, datanodeUuid=8339f64a-a65e-4a7c-aeab-6bb7febc774a, infoPort=34371, infoSecurePort=0, ipcPort=39577, storageInfo=lv=-57;cid=testClusterID;nsid=678129203;c=1732422620375), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T04:30:34,531 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@e87d64e{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f4065fa-2df8-4567-fbb0-481a3f88ce22/java.io.tmpdir/jetty-localhost-35717-hadoop-hdfs-3_4_1-tests_jar-_-any-9472547181933442366/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T04:30:34,531 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3928e3c{HTTP/1.1, (http/1.1)}{localhost:35717}
2024-11-24T04:30:34,531 INFO  [Time-limited test {}] server.Server(415): Started @163593ms
2024-11-24T04:30:34,533 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-24T04:30:34,621 WARN  [Thread-1371 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-24T04:30:34,623 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb47de63eee0e6037 with lease ID 0xff875ab8a6d51711: from storage DS-38ec15dd-c607-46cb-8f20-43e17ecaf8ee node DatanodeRegistration(127.0.0.1:43783, datanodeUuid=5e400a8b-e68b-4fbe-8406-f707b7cad460, infoPort=34729, infoSecurePort=0, ipcPort=44957, storageInfo=lv=-57;cid=testClusterID;nsid=678129203;c=1732422620375), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T04:30:34,623 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb47de63eee0e6037 with lease ID 0xff875ab8a6d51711: from storage DS-40d508c5-83fc-4808-89de-d8587e88042a node DatanodeRegistration(127.0.0.1:43783, datanodeUuid=5e400a8b-e68b-4fbe-8406-f707b7cad460, infoPort=34729, infoSecurePort=0, ipcPort=44957, storageInfo=lv=-57;cid=testClusterID;nsid=678129203;c=1732422620375), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T04:30:34,767 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:34,807 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:35,550 INFO  [Time-limited test {}] wal.TestLogRolling(372): Data Nodes restarted
2024-11-24T04:30:35,553 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1002
2024-11-24T04:30:35,554 ERROR [FSHLog-0-hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf-prefix:fadb2fe0ba32,35619,1732422621246 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41397,DS-38ec15dd-c607-46cb-8f20-43e17ecaf8ee,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:30:35,554 WARN  [FSHLog-0-hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf-prefix:fadb2fe0ba32,35619,1732422621246 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41397,DS-38ec15dd-c607-46cb-8f20-43e17ecaf8ee,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:30:35,554 DEBUG [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog fadb2fe0ba32%2C35619%2C1732422621246:(num 1732422621649) roll requested
2024-11-24T04:30:35,554 INFO  [regionserver/fadb2fe0ba32:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor fadb2fe0ba32%2C35619%2C1732422621246.1732422635554
2024-11-24T04:30:35,560 DEBUG [regionserver/fadb2fe0ba32:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.1732422621649 newFile=hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.1732422635554
2024-11-24T04:30:35,560 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:35,560 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:35,560 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:35,560 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:35,560 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:35,560 INFO  [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.1732422621649 with entries=2, filesize=1.59 KB; new WAL /user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.1732422635554
2024-11-24T04:30:35,561 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41397,DS-38ec15dd-c607-46cb-8f20-43e17ecaf8ee,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:30:35,561 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41397,DS-38ec15dd-c607-46cb-8f20-43e17ecaf8ee,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:30:35,561 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.1732422621649
2024-11-24T04:30:35,561 DEBUG [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34729:34729),(127.0.0.1/127.0.0.1:34371:34371)]
2024-11-24T04:30:35,561 DEBUG [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.1732422621649 is not closed yet, will try archiving it next time
2024-11-24T04:30:35,561 WARN  [IPC Server handler 1 on default port 38691 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.1732422621649 has not been closed. Lease recovery is in progress. RecoveryId = 1017 for block blk_1073741833_1014
2024-11-24T04:30:35,562 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.1732422621649 after 1ms
2024-11-24T04:30:35,624 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43783 is added to blk_1073741833_1017 (size=1632)
2024-11-24T04:30:35,768 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:35,808 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:36,768 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:36,808 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:37,565 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1003
2024-11-24T04:30:37,769 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:37,809 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:38,769 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:38,809 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:39,470 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741833_1014: GenerationStamp not matched, existing replica is blk_1073741833_1009
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-11-24T04:30:39,562 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.1732422621649 after 4001ms
2024-11-24T04:30:39,567 WARN  [ResponseProcessor for block BP-878497079-172.17.0.2-1732422620375:blk_1073741837_1016 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-878497079-172.17.0.2-1732422620375:blk_1073741837_1016
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:30:39,568 WARN  [DataStreamer for file /user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.1732422635554 block BP-878497079-172.17.0.2-1732422620375:blk_1073741837_1016 {}] hdfs.DataStreamer(1731): Error Recovery for BP-878497079-172.17.0.2-1732422620375:blk_1073741837_1016 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43783,DS-38ec15dd-c607-46cb-8f20-43e17ecaf8ee,DISK], DatanodeInfoWithStorage[127.0.0.1:41225,DS-3b0df271-2496-41f3-863f-8757c97002a8,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:43783,DS-38ec15dd-c607-46cb-8f20-43e17ecaf8ee,DISK]) is bad.
2024-11-24T04:30:39,568 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_15500758_22 at /127.0.0.1:41720 [Receiving block BP-878497079-172.17.0.2-1732422620375:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:43783:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41720 dst: /127.0.0.1:43783
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T04:30:39,568 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_15500758_22 at /127.0.0.1:36342 [Receiving block BP-878497079-172.17.0.2-1732422620375:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:41225:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:36342 dst: /127.0.0.1:41225
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T04:30:39,569 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@e87d64e{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T04:30:39,570 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3928e3c{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-24T04:30:39,570 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-24T04:30:39,570 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2e49578b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-24T04:30:39,570 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@43a7cd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f4065fa-2df8-4567-fbb0-481a3f88ce22/hadoop.log.dir/,STOPPED}
2024-11-24T04:30:39,571 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-24T04:30:39,571 WARN  [BP-878497079-172.17.0.2-1732422620375 heartbeating to localhost/127.0.0.1:38691 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-24T04:30:39,571 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-24T04:30:39,571 WARN  [BP-878497079-172.17.0.2-1732422620375 heartbeating to localhost/127.0.0.1:38691 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-878497079-172.17.0.2-1732422620375 (Datanode Uuid 5e400a8b-e68b-4fbe-8406-f707b7cad460) service to localhost/127.0.0.1:38691
2024-11-24T04:30:39,572 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f4065fa-2df8-4567-fbb0-481a3f88ce22/cluster_7730fbad-df8e-ea9e-97b3-261c7f308fa8/data/data1/current/BP-878497079-172.17.0.2-1732422620375 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T04:30:39,572 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f4065fa-2df8-4567-fbb0-481a3f88ce22/cluster_7730fbad-df8e-ea9e-97b3-261c7f308fa8/data/data2/current/BP-878497079-172.17.0.2-1732422620375 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T04:30:39,572 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-24T04:30:39,582 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T04:30:39,584 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-24T04:30:39,587 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-24T04:30:39,587 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-24T04:30:39,587 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-24T04:30:39,587 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@57d5f4b3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f4065fa-2df8-4567-fbb0-481a3f88ce22/hadoop.log.dir/,AVAILABLE}
2024-11-24T04:30:39,588 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@fa662a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-24T04:30:39,701 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@290ac13e{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f4065fa-2df8-4567-fbb0-481a3f88ce22/java.io.tmpdir/jetty-localhost-40033-hadoop-hdfs-3_4_1-tests_jar-_-any-16424801687220755969/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T04:30:39,702 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@117b424d{HTTP/1.1, (http/1.1)}{localhost:40033}
2024-11-24T04:30:39,702 INFO  [Time-limited test {}] server.Server(415): Started @168763ms
2024-11-24T04:30:39,703 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-24T04:30:39,724 WARN  [ResponseProcessor for block BP-878497079-172.17.0.2-1732422620375:blk_1073741837_1018 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-878497079-172.17.0.2-1732422620375:blk_1073741837_1018
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:30:39,725 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_15500758_22 at /127.0.0.1:36352 [Receiving block BP-878497079-172.17.0.2-1732422620375:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:41225:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:36352 dst: /127.0.0.1:41225
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T04:30:39,729 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@55cf8f7d{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T04:30:39,729 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3de5676f{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-24T04:30:39,729 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-24T04:30:39,730 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@126bd190{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-24T04:30:39,730 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@36974255{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f4065fa-2df8-4567-fbb0-481a3f88ce22/hadoop.log.dir/,STOPPED}
2024-11-24T04:30:39,731 WARN  [BP-878497079-172.17.0.2-1732422620375 heartbeating to localhost/127.0.0.1:38691 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-24T04:30:39,731 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-24T04:30:39,731 WARN  [BP-878497079-172.17.0.2-1732422620375 heartbeating to localhost/127.0.0.1:38691 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-878497079-172.17.0.2-1732422620375 (Datanode Uuid 8339f64a-a65e-4a7c-aeab-6bb7febc774a) service to localhost/127.0.0.1:38691
2024-11-24T04:30:39,731 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-24T04:30:39,732 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f4065fa-2df8-4567-fbb0-481a3f88ce22/cluster_7730fbad-df8e-ea9e-97b3-261c7f308fa8/data/data3/current/BP-878497079-172.17.0.2-1732422620375 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T04:30:39,733 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f4065fa-2df8-4567-fbb0-481a3f88ce22/cluster_7730fbad-df8e-ea9e-97b3-261c7f308fa8/data/data4/current/BP-878497079-172.17.0.2-1732422620375 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T04:30:39,733 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-24T04:30:39,742 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T04:30:39,746 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-24T04:30:39,747 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-24T04:30:39,747 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-24T04:30:39,747 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-24T04:30:39,747 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@64685bd7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f4065fa-2df8-4567-fbb0-481a3f88ce22/hadoop.log.dir/,AVAILABLE}
2024-11-24T04:30:39,748 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@20466b6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-24T04:30:39,800 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:39,810 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:39,827 WARN  [Thread-1414 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-24T04:30:39,830 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbdcdd4204231f2cb with lease ID 0xff875ab8a6d51712: from storage DS-38ec15dd-c607-46cb-8f20-43e17ecaf8ee node DatanodeRegistration(127.0.0.1:34945, datanodeUuid=5e400a8b-e68b-4fbe-8406-f707b7cad460, infoPort=37895, infoSecurePort=0, ipcPort=38257, storageInfo=lv=-57;cid=testClusterID;nsid=678129203;c=1732422620375), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T04:30:39,830 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbdcdd4204231f2cb with lease ID 0xff875ab8a6d51712: from storage DS-40d508c5-83fc-4808-89de-d8587e88042a node DatanodeRegistration(127.0.0.1:34945, datanodeUuid=5e400a8b-e68b-4fbe-8406-f707b7cad460, infoPort=37895, infoSecurePort=0, ipcPort=38257, storageInfo=lv=-57;cid=testClusterID;nsid=678129203;c=1732422620375), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T04:30:39,891 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3172a7c6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f4065fa-2df8-4567-fbb0-481a3f88ce22/java.io.tmpdir/jetty-localhost-44745-hadoop-hdfs-3_4_1-tests_jar-_-any-17846960560671082963/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T04:30:39,891 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7b559376{HTTP/1.1, (http/1.1)}{localhost:44745}
2024-11-24T04:30:39,891 INFO  [Time-limited test {}] server.Server(415): Started @168953ms
2024-11-24T04:30:39,893 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-24T04:30:39,974 WARN  [Thread-1445 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-24T04:30:39,976 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb2868a0bbb1b32e2 with lease ID 0xff875ab8a6d51713: from storage DS-3b0df271-2496-41f3-863f-8757c97002a8 node DatanodeRegistration(127.0.0.1:35025, datanodeUuid=8339f64a-a65e-4a7c-aeab-6bb7febc774a, infoPort=36131, infoSecurePort=0, ipcPort=46289, storageInfo=lv=-57;cid=testClusterID;nsid=678129203;c=1732422620375), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T04:30:39,976 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb2868a0bbb1b32e2 with lease ID 0xff875ab8a6d51713: from storage DS-3eb47113-507d-4e64-99e0-a84100107583 node DatanodeRegistration(127.0.0.1:35025, datanodeUuid=8339f64a-a65e-4a7c-aeab-6bb7febc774a, infoPort=36131, infoSecurePort=0, ipcPort=46289, storageInfo=lv=-57;cid=testClusterID;nsid=678129203;c=1732422620375), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T04:30:40,801 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:40,810 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:40,910 INFO  [Time-limited test {}] wal.TestLogRolling(389): Data Nodes restarted
2024-11-24T04:30:40,913 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1004
2024-11-24T04:30:40,914 ERROR [FSHLog-0-hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf-prefix:fadb2fe0ba32,35619,1732422621246 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41225,DS-3b0df271-2496-41f3-863f-8757c97002a8,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:30:40,914 WARN  [FSHLog-0-hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf-prefix:fadb2fe0ba32,35619,1732422621246 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41225,DS-3b0df271-2496-41f3-863f-8757c97002a8,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:30:40,914 DEBUG [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog fadb2fe0ba32%2C35619%2C1732422621246:(num 1732422635554) roll requested
2024-11-24T04:30:40,914 INFO  [regionserver/fadb2fe0ba32:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor fadb2fe0ba32%2C35619%2C1732422621246.1732422640914
2024-11-24T04:30:40,920 DEBUG [regionserver/fadb2fe0ba32:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.1732422635554 newFile=hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.1732422640914
2024-11-24T04:30:40,920 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:40,920 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:40,920 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:40,920 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:40,920 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:40,920 INFO  [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.1732422635554 with entries=2, filesize=2.37 KB; new WAL /user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.1732422640914
2024-11-24T04:30:40,921 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41225,DS-3b0df271-2496-41f3-863f-8757c97002a8,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:30:40,921 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41225,DS-3b0df271-2496-41f3-863f-8757c97002a8,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:30:40,921 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.1732422635554
2024-11-24T04:30:40,921 DEBUG [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37895:37895),(127.0.0.1/127.0.0.1:36131:36131)]
2024-11-24T04:30:40,921 DEBUG [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.1732422635554 is not closed yet, will try archiving it next time
2024-11-24T04:30:40,921 WARN  [IPC Server handler 4 on default port 38691 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.1732422635554 has not been closed. Lease recovery is in progress. RecoveryId = 1020 for block blk_1073741837_1018
2024-11-24T04:30:40,922 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.1732422635554 after 0ms
2024-11-24T04:30:41,802 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:41,811 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:41,830 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741837_1018: GenerationStamp not matched, existing replica is blk_1073741837_1016
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-11-24T04:30:42,802 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:42,812 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:42,923 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor fadb2fe0ba32%2C35619%2C1732422621246.1732422642922
2024-11-24T04:30:42,928 DEBUG [Time-limited test {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.1732422640914 newFile=hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.1732422642922
2024-11-24T04:30:42,928 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:42,928 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:42,929 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:42,929 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:42,929 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:42,929 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.1732422640914 with entries=1, filesize=1.23 KB; new WAL /user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.1732422642922
2024-11-24T04:30:42,930 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36131:36131),(127.0.0.1/127.0.0.1:37895:37895)]
2024-11-24T04:30:42,930 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.1732422635554 is not closed yet, will try archiving it next time
2024-11-24T04:30:42,930 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.1732422640914 is not closed yet, will try archiving it next time
2024-11-24T04:30:42,930 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.1732422621649
2024-11-24T04:30:42,930 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.1732422621649
2024-11-24T04:30:42,931 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.1732422621649 after 1ms
2024-11-24T04:30:42,931 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35025 is added to blk_1073741838_1019 (size=1264)
2024-11-24T04:30:42,931 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.1732422621649
2024-11-24T04:30:42,931 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34945 is added to blk_1073741838_1019 (size=1264)
2024-11-24T04:30:42,932 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.1732422635554 is not closed yet, will try archiving it next time
2024-11-24T04:30:42,942 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #3: [\x00/METAFAMILY:HBASE::REGION_EVENT::REGION_OPEN/1732422622543/Put/vlen=218/seqid=0]
2024-11-24T04:30:42,943 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #4: [row1002/info:/1732422632221/Put/vlen=1045/seqid=0]
2024-11-24T04:30:42,943 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.1732422621649
2024-11-24T04:30:42,943 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.1732422635554
2024-11-24T04:30:42,943 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.1732422635554
2024-11-24T04:30:42,944 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.1732422635554 after 0ms
2024-11-24T04:30:42,944 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.1732422635554
2024-11-24T04:30:42,947 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #5: [row1003/info:/1732422635554/Put/vlen=1045/seqid=0]
2024-11-24T04:30:42,948 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #6: [row1004/info:/1732422637565/Put/vlen=1045/seqid=0]
2024-11-24T04:30:42,948 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.1732422635554
2024-11-24T04:30:42,948 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.1732422640914
2024-11-24T04:30:42,948 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.1732422640914
2024-11-24T04:30:42,948 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.1732422640914 after 0ms
2024-11-24T04:30:42,948 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.1732422640914
2024-11-24T04:30:42,952 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #7: [row1005/info:/1732422640914/Put/vlen=1045/seqid=0]
2024-11-24T04:30:42,952 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.1732422642922
2024-11-24T04:30:42,952 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.1732422642922
2024-11-24T04:30:42,952 WARN  [IPC Server handler 0 on default port 38691 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.1732422642922 has not been closed. Lease recovery is in progress. RecoveryId = 1022 for block blk_1073741839_1021
2024-11-24T04:30:42,952 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.1732422642922 after 0ms
2024-11-24T04:30:43,803 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:43,812 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:43,834 WARN  [ResponseProcessor for block BP-878497079-172.17.0.2-1732422620375:blk_1073741839_1021 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-878497079-172.17.0.2-1732422620375:blk_1073741839_1021
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:30:43,834 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-329496266_22 at /127.0.0.1:54308 [Receiving block BP-878497079-172.17.0.2-1732422620375:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:34945:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:54308 dst: /127.0.0.1:34945
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T04:30:43,834 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-329496266_22 at /127.0.0.1:59612 [Receiving block BP-878497079-172.17.0.2-1732422620375:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:35025:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:59612 dst: /127.0.0.1:35025
java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:35025 remote=/127.0.0.1:59612]. Total timeout mills is 60000, 59094 millis timeout left.
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T04:30:43,834 WARN  [DataStreamer for file /user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.1732422642922 block BP-878497079-172.17.0.2-1732422620375:blk_1073741839_1021 {}] hdfs.DataStreamer(1731): Error Recovery for BP-878497079-172.17.0.2-1732422620375:blk_1073741839_1021 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35025,DS-3b0df271-2496-41f3-863f-8757c97002a8,DISK], DatanodeInfoWithStorage[127.0.0.1:34945,DS-38ec15dd-c607-46cb-8f20-43e17ecaf8ee,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35025,DS-3b0df271-2496-41f3-863f-8757c97002a8,DISK]) is bad.
2024-11-24T04:30:43,835 WARN  [DataStreamer for file /user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.1732422642922 block BP-878497079-172.17.0.2-1732422620375:blk_1073741839_1021 {}] hdfs.DataStreamer(859): DataStreamer Exception
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-878497079-172.17.0.2-1732422620375:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor113.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor113.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor113.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor113.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor113.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor113.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor113.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:30:43,839 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35025 is added to blk_1073741839_1022 (size=85)
2024-11-24T04:30:44,803 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:44,813 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:44,923 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.1732422635554 after 4001ms
2024-11-24T04:30:45,804 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:45,813 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:46,805 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:46,814 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:46,953 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.1732422642922 after 4001ms
2024-11-24T04:30:46,953 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.1732422642922
2024-11-24T04:30:46,957 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.1732422642922
2024-11-24T04:30:46,957 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.74 KB heapSize=3.77 KB
2024-11-24T04:30:46,958 ERROR [FSHLog-0-hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf-prefix:fadb2fe0ba32,35619,1732422621246.meta {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41397,DS-38ec15dd-c607-46cb-8f20-43e17ecaf8ee,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:30:46,958 WARN  [FSHLog-0-hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf-prefix:fadb2fe0ba32,35619,1732422621246.meta {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41397,DS-38ec15dd-c607-46cb-8f20-43e17ecaf8ee,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:30:46,958 DEBUG [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog fadb2fe0ba32%2C35619%2C1732422621246.meta:.meta(num 1732422622057) roll requested
2024-11-24T04:30:46,959 INFO  [regionserver/fadb2fe0ba32:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor fadb2fe0ba32%2C35619%2C1732422621246.meta.1732422646958.meta
2024-11-24T04:30:46,964 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:46,964 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:46,964 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:46,964 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:46,964 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:46,964 INFO  [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.meta.1732422622057.meta with entries=8, filesize=2.36 KB; new WAL /user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.meta.1732422646958.meta
2024-11-24T04:30:46,965 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41397,DS-38ec15dd-c607-46cb-8f20-43e17ecaf8ee,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:30:46,965 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41397,DS-38ec15dd-c607-46cb-8f20-43e17ecaf8ee,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:30:46,965 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.meta.1732422622057.meta
2024-11-24T04:30:46,965 WARN  [IPC Server handler 3 on default port 38691 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.meta.1732422622057.meta has not been closed. Lease recovery is in progress. RecoveryId = 1024 for block blk_1073741834_1013
2024-11-24T04:30:46,965 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.meta.1732422622057.meta after 0ms
2024-11-24T04:30:46,968 DEBUG [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37895:37895),(127.0.0.1/127.0.0.1:36131:36131)]
2024-11-24T04:30:46,968 DEBUG [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.meta.1732422622057.meta is not closed yet, will try archiving it next time
2024-11-24T04:30:46,992 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/data/hbase/meta/1588230740/.tmp/info/147fdc8c024c4f88b314976eeff7ce29 is 207, key is TestLogRolling-testLogRollOnPipelineRestart,,1732422622182.8f02b50dcfd7865c201db72b9bcecab1./info:regioninfo/1732422622547/Put/seqid=0
2024-11-24T04:30:46,997 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34945 is added to blk_1073741841_1025 (size=7125)
2024-11-24T04:30:46,997 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35025 is added to blk_1073741841_1025 (size=7125)
2024-11-24T04:30:47,398 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.52 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/data/hbase/meta/1588230740/.tmp/info/147fdc8c024c4f88b314976eeff7ce29
2024-11-24T04:30:47,426 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/data/hbase/meta/1588230740/.tmp/ns/463fa50ddd4a4593a252697207d31384 is 43, key is default/ns:d/1732422622108/Put/seqid=0
2024-11-24T04:30:47,431 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35025 is added to blk_1073741842_1026 (size=5153)
2024-11-24T04:30:47,431 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34945 is added to blk_1073741842_1026 (size=5153)
2024-11-24T04:30:47,432 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/data/hbase/meta/1588230740/.tmp/ns/463fa50ddd4a4593a252697207d31384
2024-11-24T04:30:47,452 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/data/hbase/meta/1588230740/.tmp/table/82687e06948d432992ec30504d32c2a7 is 79, key is TestLogRolling-testLogRollOnPipelineRestart/table:state/1732422622558/Put/seqid=0
2024-11-24T04:30:47,457 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34945 is added to blk_1073741843_1027 (size=5438)
2024-11-24T04:30:47,457 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35025 is added to blk_1073741843_1027 (size=5438)
2024-11-24T04:30:47,457 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=150 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/data/hbase/meta/1588230740/.tmp/table/82687e06948d432992ec30504d32c2a7
2024-11-24T04:30:47,463 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/data/hbase/meta/1588230740/.tmp/info/147fdc8c024c4f88b314976eeff7ce29 as hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/data/hbase/meta/1588230740/info/147fdc8c024c4f88b314976eeff7ce29
2024-11-24T04:30:47,468 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/data/hbase/meta/1588230740/info/147fdc8c024c4f88b314976eeff7ce29, entries=10, sequenceid=11, filesize=7.0 K
2024-11-24T04:30:47,469 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/data/hbase/meta/1588230740/.tmp/ns/463fa50ddd4a4593a252697207d31384 as hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/data/hbase/meta/1588230740/ns/463fa50ddd4a4593a252697207d31384
2024-11-24T04:30:47,474 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/data/hbase/meta/1588230740/ns/463fa50ddd4a4593a252697207d31384, entries=2, sequenceid=11, filesize=5.0 K
2024-11-24T04:30:47,475 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/data/hbase/meta/1588230740/.tmp/table/82687e06948d432992ec30504d32c2a7 as hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/data/hbase/meta/1588230740/table/82687e06948d432992ec30504d32c2a7
2024-11-24T04:30:47,481 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/data/hbase/meta/1588230740/table/82687e06948d432992ec30504d32c2a7, entries=2, sequenceid=11, filesize=5.3 K
2024-11-24T04:30:47,482 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.74 KB/1782, heapSize ~3.48 KB/3560, currentSize=0 B/0 for 1588230740 in 525ms, sequenceid=11, compaction requested=false
2024-11-24T04:30:47,482 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1588230740:

2024-11-24T04:30:47,482 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 8f02b50dcfd7865c201db72b9bcecab1 1/1 column families, dataSize=4.20 KB heapSize=4.75 KB
2024-11-24T04:30:47,483 ERROR [FSHLog-0-hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf-prefix:fadb2fe0ba32,35619,1732422621246 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-878497079-172.17.0.2-1732422620375:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor113.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor113.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor113.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor113.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor113.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor113.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor113.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:30:47,483 WARN  [FSHLog-0-hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf-prefix:fadb2fe0ba32,35619,1732422621246 {}] wal.AbstractFSWAL(2174): append entry failed
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-878497079-172.17.0.2-1732422620375:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor113.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor113.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor113.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor113.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor113.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor113.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor113.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:30:47,483 DEBUG [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog fadb2fe0ba32%2C35619%2C1732422621246:(num 1732422642922) roll requested
2024-11-24T04:30:47,484 INFO  [regionserver/fadb2fe0ba32:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor fadb2fe0ba32%2C35619%2C1732422621246.1732422647484
2024-11-24T04:30:47,488 DEBUG [regionserver/fadb2fe0ba32:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.1732422642922 newFile=hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.1732422647484
2024-11-24T04:30:47,488 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:47,489 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:47,489 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:47,489 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:47,489 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:47,489 INFO  [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.1732422642922 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.1732422647484
2024-11-24T04:30:47,489 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-878497079-172.17.0.2-1732422620375:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor113.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor113.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor113.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor113.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor113.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor113.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor113.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:30:47,490 DEBUG [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36131:36131),(127.0.0.1/127.0.0.1:37895:37895)]
2024-11-24T04:30:47,490 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-878497079-172.17.0.2-1732422620375:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor113.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor113.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor113.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor113.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor113.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor113.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor113.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:30:47,490 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.1732422642922
2024-11-24T04:30:47,490 DEBUG [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.1732422642922 is not closed yet, will try archiving it next time
2024-11-24T04:30:47,490 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.1732422642922 after 0ms
2024-11-24T04:30:47,491 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.1732422642922 to hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/oldWALs/fadb2fe0ba32%2C35619%2C1732422621246.1732422642922
2024-11-24T04:30:47,505 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/data/default/TestLogRolling-testLogRollOnPipelineRestart/8f02b50dcfd7865c201db72b9bcecab1/.tmp/info/905470c54a30433ea129b807583c6348 is 1080, key is row1002/info:/1732422632221/Put/seqid=0
2024-11-24T04:30:47,510 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34945 is added to blk_1073741845_1029 (size=9270)
2024-11-24T04:30:47,510 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35025 is added to blk_1073741845_1029 (size=9270)
2024-11-24T04:30:47,511 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.20 KB at sequenceid=8 (bloomFilter=true), to=hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/data/default/TestLogRolling-testLogRollOnPipelineRestart/8f02b50dcfd7865c201db72b9bcecab1/.tmp/info/905470c54a30433ea129b807583c6348
2024-11-24T04:30:47,517 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/data/default/TestLogRolling-testLogRollOnPipelineRestart/8f02b50dcfd7865c201db72b9bcecab1/.tmp/info/905470c54a30433ea129b807583c6348 as hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/data/default/TestLogRolling-testLogRollOnPipelineRestart/8f02b50dcfd7865c201db72b9bcecab1/info/905470c54a30433ea129b807583c6348
2024-11-24T04:30:47,522 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/data/default/TestLogRolling-testLogRollOnPipelineRestart/8f02b50dcfd7865c201db72b9bcecab1/info/905470c54a30433ea129b807583c6348, entries=4, sequenceid=8, filesize=9.1 K
2024-11-24T04:30:47,524 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~4.20 KB/4304, heapSize ~4.73 KB/4848, currentSize=0 B/0 for 8f02b50dcfd7865c201db72b9bcecab1 in 41ms, sequenceid=8, compaction requested=false
2024-11-24T04:30:47,524 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 8f02b50dcfd7865c201db72b9bcecab1:

2024-11-24T04:30:47,529 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-24T04:30:47,529 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-24T04:30:47,529 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-24T04:30:47,529 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T04:30:47,529 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T04:30:47,529 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-24T04:30:47,529 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-24T04:30:47,529 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1746566449, stopped=false
2024-11-24T04:30:47,529 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=fadb2fe0ba32,39473,1732422621180
2024-11-24T04:30:47,531 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39473-0x101483d427d0000, quorum=127.0.0.1:51577, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-24T04:30:47,531 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35619-0x101483d427d0001, quorum=127.0.0.1:51577, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-24T04:30:47,531 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39473-0x101483d427d0000, quorum=127.0.0.1:51577, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:30:47,531 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35619-0x101483d427d0001, quorum=127.0.0.1:51577, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:30:47,531 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-24T04:30:47,531 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-24T04:30:47,532 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-24T04:30:47,532 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T04:30:47,532 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'fadb2fe0ba32,35619,1732422621246' *****
2024-11-24T04:30:47,532 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39473-0x101483d427d0000, quorum=127.0.0.1:51577, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-24T04:30:47,532 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-24T04:30:47,532 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35619-0x101483d427d0001, quorum=127.0.0.1:51577, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-24T04:30:47,532 INFO  [RS:0;fadb2fe0ba32:35619 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-24T04:30:47,532 INFO  [RS:0;fadb2fe0ba32:35619 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-24T04:30:47,532 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-24T04:30:47,532 INFO  [RS:0;fadb2fe0ba32:35619 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-24T04:30:47,532 INFO  [RS:0;fadb2fe0ba32:35619 {}] regionserver.HRegionServer(3091): Received CLOSE for 8f02b50dcfd7865c201db72b9bcecab1
2024-11-24T04:30:47,533 INFO  [RS:0;fadb2fe0ba32:35619 {}] regionserver.HRegionServer(959): stopping server fadb2fe0ba32,35619,1732422621246
2024-11-24T04:30:47,533 INFO  [RS:0;fadb2fe0ba32:35619 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-24T04:30:47,533 INFO  [RS:0;fadb2fe0ba32:35619 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;fadb2fe0ba32:35619.
2024-11-24T04:30:47,533 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8f02b50dcfd7865c201db72b9bcecab1, disabling compactions & flushes
2024-11-24T04:30:47,533 INFO  [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1732422622182.8f02b50dcfd7865c201db72b9bcecab1.
2024-11-24T04:30:47,533 DEBUG [RS:0;fadb2fe0ba32:35619 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-24T04:30:47,533 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1732422622182.8f02b50dcfd7865c201db72b9bcecab1.
2024-11-24T04:30:47,533 DEBUG [RS:0;fadb2fe0ba32:35619 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T04:30:47,533 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1732422622182.8f02b50dcfd7865c201db72b9bcecab1. after waiting 0 ms
2024-11-24T04:30:47,533 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1732422622182.8f02b50dcfd7865c201db72b9bcecab1.
2024-11-24T04:30:47,533 INFO  [RS:0;fadb2fe0ba32:35619 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-24T04:30:47,533 INFO  [RS:0;fadb2fe0ba32:35619 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-24T04:30:47,533 INFO  [RS:0;fadb2fe0ba32:35619 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-24T04:30:47,533 INFO  [RS:0;fadb2fe0ba32:35619 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-24T04:30:47,534 INFO  [RS:0;fadb2fe0ba32:35619 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-11-24T04:30:47,534 DEBUG [RS:0;fadb2fe0ba32:35619 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 8f02b50dcfd7865c201db72b9bcecab1=TestLogRolling-testLogRollOnPipelineRestart,,1732422622182.8f02b50dcfd7865c201db72b9bcecab1.}
2024-11-24T04:30:47,534 DEBUG [RS:0;fadb2fe0ba32:35619 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 8f02b50dcfd7865c201db72b9bcecab1
2024-11-24T04:30:47,534 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-24T04:30:47,534 INFO  [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-24T04:30:47,534 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-24T04:30:47,534 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-24T04:30:47,534 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-24T04:30:47,538 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/data/default/TestLogRolling-testLogRollOnPipelineRestart/8f02b50dcfd7865c201db72b9bcecab1/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1
2024-11-24T04:30:47,539 INFO  [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnPipelineRestart,,1732422622182.8f02b50dcfd7865c201db72b9bcecab1.
2024-11-24T04:30:47,539 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8f02b50dcfd7865c201db72b9bcecab1:
Waiting for close lock at 1732422647533Running coprocessor pre-close hooks at 1732422647533Disabling compacts and flushes for region at 1732422647533Disabling writes for close at 1732422647533Writing region close event to WAL at 1732422647534 (+1 ms)Running coprocessor post-close hooks at 1732422647539 (+5 ms)Closed at 1732422647539
2024-11-24T04:30:47,539 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnPipelineRestart,,1732422622182.8f02b50dcfd7865c201db72b9bcecab1.
2024-11-24T04:30:47,539 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-11-24T04:30:47,540 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-24T04:30:47,540 INFO  [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-24T04:30:47,540 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732422647534Running coprocessor pre-close hooks at 1732422647534Disabling compacts and flushes for region at 1732422647534Disabling writes for close at 1732422647534Writing region close event to WAL at 1732422647535 (+1 ms)Running coprocessor post-close hooks at 1732422647540 (+5 ms)Closed at 1732422647540
2024-11-24T04:30:47,540 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-24T04:30:47,734 INFO  [RS:0;fadb2fe0ba32:35619 {}] regionserver.HRegionServer(976): stopping server fadb2fe0ba32,35619,1732422621246; all regions closed.
2024-11-24T04:30:47,734 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:47,735 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:47,735 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:47,735 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:47,735 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:47,737 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35025 is added to blk_1073741840_1023 (size=825)
2024-11-24T04:30:47,737 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34945 is added to blk_1073741840_1023 (size=825)
2024-11-24T04:30:47,805 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:47,814 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:48,515 INFO  [regionserver/fadb2fe0ba32:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-11-24T04:30:48,515 INFO  [regionserver/fadb2fe0ba32:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-11-24T04:30:48,806 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:48,815 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:49,514 INFO  [regionserver/fadb2fe0ba32:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-24T04:30:49,806 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:49,815 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:50,807 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:50,815 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:50,966 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.meta.1732422622057.meta after 4001ms
2024-11-24T04:30:50,967 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/WALs/fadb2fe0ba32,35619,1732422621246/fadb2fe0ba32%2C35619%2C1732422621246.meta.1732422622057.meta to hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/oldWALs/fadb2fe0ba32%2C35619%2C1732422621246.meta.1732422622057.meta
2024-11-24T04:30:50,970 DEBUG [RS:0;fadb2fe0ba32:35619 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/oldWALs
2024-11-24T04:30:50,970 INFO  [RS:0;fadb2fe0ba32:35619 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog fadb2fe0ba32%2C35619%2C1732422621246.meta:.meta(num 1732422646958)
2024-11-24T04:30:50,970 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:50,970 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:50,970 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:50,970 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:50,970 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:50,972 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34945 is added to blk_1073741844_1028 (size=1162)
2024-11-24T04:30:50,973 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35025 is added to blk_1073741844_1028 (size=1162)
2024-11-24T04:30:50,977 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741834_1013: GenerationStamp not matched, existing replica is blk_1073741834_1010
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-11-24T04:30:50,977 DEBUG [RS:0;fadb2fe0ba32:35619 {}] wal.AbstractFSWAL(1256): Moved 4 WAL file(s) to /user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/oldWALs
2024-11-24T04:30:50,977 INFO  [RS:0;fadb2fe0ba32:35619 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog fadb2fe0ba32%2C35619%2C1732422621246:(num 1732422647484)
2024-11-24T04:30:50,977 DEBUG [RS:0;fadb2fe0ba32:35619 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T04:30:50,977 INFO  [RS:0;fadb2fe0ba32:35619 {}] regionserver.LeaseManager(133): Closed leases
2024-11-24T04:30:50,977 INFO  [RS:0;fadb2fe0ba32:35619 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-24T04:30:50,977 INFO  [RS:0;fadb2fe0ba32:35619 {}] hbase.ChoreService(370): Chore service for: regionserver/fadb2fe0ba32:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown
2024-11-24T04:30:50,977 INFO  [RS:0;fadb2fe0ba32:35619 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-24T04:30:50,977 INFO  [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-24T04:30:50,978 INFO  [RS:0;fadb2fe0ba32:35619 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35619
2024-11-24T04:30:50,979 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35619-0x101483d427d0001, quorum=127.0.0.1:51577, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/fadb2fe0ba32,35619,1732422621246
2024-11-24T04:30:50,979 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39473-0x101483d427d0000, quorum=127.0.0.1:51577, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-24T04:30:50,980 INFO  [RS:0;fadb2fe0ba32:35619 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-24T04:30:50,981 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [fadb2fe0ba32,35619,1732422621246]
2024-11-24T04:30:50,982 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/fadb2fe0ba32,35619,1732422621246 already deleted, retry=false
2024-11-24T04:30:50,982 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; fadb2fe0ba32,35619,1732422621246 expired; onlineServers=0
2024-11-24T04:30:50,982 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'fadb2fe0ba32,39473,1732422621180' *****
2024-11-24T04:30:50,982 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-24T04:30:50,983 INFO  [M:0;fadb2fe0ba32:39473 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-24T04:30:50,983 INFO  [M:0;fadb2fe0ba32:39473 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-24T04:30:50,983 DEBUG [M:0;fadb2fe0ba32:39473 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-24T04:30:50,983 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-24T04:30:50,983 DEBUG [M:0;fadb2fe0ba32:39473 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-24T04:30:50,983 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster-HFileCleaner.small.0-1732422621431 {}] cleaner.HFileCleaner(306): Exit Thread[master/fadb2fe0ba32:0:becomeActiveMaster-HFileCleaner.small.0-1732422621431,5,FailOnTimeoutGroup]
2024-11-24T04:30:50,983 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster-HFileCleaner.large.0-1732422621431 {}] cleaner.HFileCleaner(306): Exit Thread[master/fadb2fe0ba32:0:becomeActiveMaster-HFileCleaner.large.0-1732422621431,5,FailOnTimeoutGroup]
2024-11-24T04:30:50,983 INFO  [M:0;fadb2fe0ba32:39473 {}] hbase.ChoreService(370): Chore service for: master/fadb2fe0ba32:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-24T04:30:50,983 INFO  [M:0;fadb2fe0ba32:39473 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-24T04:30:50,983 DEBUG [M:0;fadb2fe0ba32:39473 {}] master.HMaster(1795): Stopping service threads
2024-11-24T04:30:50,983 INFO  [M:0;fadb2fe0ba32:39473 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-24T04:30:50,984 INFO  [M:0;fadb2fe0ba32:39473 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-24T04:30:50,984 INFO  [M:0;fadb2fe0ba32:39473 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-24T04:30:50,984 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-24T04:30:50,986 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39473-0x101483d427d0000, quorum=127.0.0.1:51577, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-24T04:30:50,986 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39473-0x101483d427d0000, quorum=127.0.0.1:51577, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:30:50,986 DEBUG [M:0;fadb2fe0ba32:39473 {}] zookeeper.ZKUtil(347): master:39473-0x101483d427d0000, quorum=127.0.0.1:51577, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-24T04:30:50,986 WARN  [M:0;fadb2fe0ba32:39473 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-24T04:30:50,987 INFO  [M:0;fadb2fe0ba32:39473 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/.lastflushedseqids
2024-11-24T04:30:50,992 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35025 is added to blk_1073741846_1030 (size=120)
2024-11-24T04:30:50,992 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34945 is added to blk_1073741846_1030 (size=120)
2024-11-24T04:30:50,992 INFO  [M:0;fadb2fe0ba32:39473 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-24T04:30:50,992 INFO  [M:0;fadb2fe0ba32:39473 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-24T04:30:50,992 DEBUG [M:0;fadb2fe0ba32:39473 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-24T04:30:50,993 INFO  [M:0;fadb2fe0ba32:39473 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T04:30:50,993 DEBUG [M:0;fadb2fe0ba32:39473 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T04:30:50,993 DEBUG [M:0;fadb2fe0ba32:39473 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-24T04:30:50,993 DEBUG [M:0;fadb2fe0ba32:39473 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T04:30:50,993 INFO  [M:0;fadb2fe0ba32:39473 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.17 KB heapSize=29.16 KB
2024-11-24T04:30:50,993 ERROR [FSHLog-0-hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/MasterData-prefix:fadb2fe0ba32,39473,1732422621180 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41397,DS-38ec15dd-c607-46cb-8f20-43e17ecaf8ee,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:30:50,993 WARN  [FSHLog-0-hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/MasterData-prefix:fadb2fe0ba32,39473,1732422621180 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41397,DS-38ec15dd-c607-46cb-8f20-43e17ecaf8ee,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:30:50,993 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL FSHLog fadb2fe0ba32%2C39473%2C1732422621180:(num 1732422621341) roll requested
2024-11-24T04:30:50,994 INFO  [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor fadb2fe0ba32%2C39473%2C1732422621180.1732422650993
2024-11-24T04:30:50,998 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:50,998 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:50,998 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:50,998 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:50,998 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:50,999 INFO  [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/MasterData/WALs/fadb2fe0ba32,39473,1732422621180/fadb2fe0ba32%2C39473%2C1732422621180.1732422621341 with entries=53, filesize=26.62 KB; new WAL /user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/MasterData/WALs/fadb2fe0ba32,39473,1732422621180/fadb2fe0ba32%2C39473%2C1732422621180.1732422650993
2024-11-24T04:30:50,999 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41397,DS-38ec15dd-c607-46cb-8f20-43e17ecaf8ee,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:30:50,999 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41397,DS-38ec15dd-c607-46cb-8f20-43e17ecaf8ee,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-24T04:30:50,999 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/MasterData/WALs/fadb2fe0ba32,39473,1732422621180/fadb2fe0ba32%2C39473%2C1732422621180.1732422621341
2024-11-24T04:30:50,999 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36131:36131),(127.0.0.1/127.0.0.1:37895:37895)]
2024-11-24T04:30:50,999 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/MasterData/WALs/fadb2fe0ba32,39473,1732422621180/fadb2fe0ba32%2C39473%2C1732422621180.1732422621341 is not closed yet, will try archiving it next time
2024-11-24T04:30:50,999 WARN  [IPC Server handler 1 on default port 38691 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/MasterData/WALs/fadb2fe0ba32,39473,1732422621180/fadb2fe0ba32%2C39473%2C1732422621180.1732422621341 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741830_1015
2024-11-24T04:30:51,000 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/MasterData/WALs/fadb2fe0ba32,39473,1732422621180/fadb2fe0ba32%2C39473%2C1732422621180.1732422621341 after 1ms
2024-11-24T04:30:51,015 DEBUG [M:0;fadb2fe0ba32:39473 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1392738e728d40418ac2cdbeaf242243 is 82, key is hbase:meta,,1/info:regioninfo/1732422622089/Put/seqid=0
2024-11-24T04:30:51,020 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35025 is added to blk_1073741848_1033 (size=5672)
2024-11-24T04:30:51,020 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34945 is added to blk_1073741848_1033 (size=5672)
2024-11-24T04:30:51,020 INFO  [M:0;fadb2fe0ba32:39473 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1392738e728d40418ac2cdbeaf242243
2024-11-24T04:30:51,040 DEBUG [M:0;fadb2fe0ba32:39473 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d327da8dff91450c9c5affc6b52f18e5 is 778, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732422622564/Put/seqid=0
2024-11-24T04:30:51,044 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35025 is added to blk_1073741849_1034 (size=6118)
2024-11-24T04:30:51,044 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34945 is added to blk_1073741849_1034 (size=6118)
2024-11-24T04:30:51,045 INFO  [M:0;fadb2fe0ba32:39473 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.57 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d327da8dff91450c9c5affc6b52f18e5
2024-11-24T04:30:51,064 DEBUG [M:0;fadb2fe0ba32:39473 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5a74581c64474ce58b5fb0354ace3be3 is 69, key is fadb2fe0ba32,35619,1732422621246/rs:state/1732422621496/Put/seqid=0
2024-11-24T04:30:51,068 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34945 is added to blk_1073741850_1035 (size=5156)
2024-11-24T04:30:51,069 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35025 is added to blk_1073741850_1035 (size=5156)
2024-11-24T04:30:51,069 INFO  [M:0;fadb2fe0ba32:39473 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5a74581c64474ce58b5fb0354ace3be3
2024-11-24T04:30:51,081 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35619-0x101483d427d0001, quorum=127.0.0.1:51577, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-24T04:30:51,081 INFO  [RS:0;fadb2fe0ba32:35619 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-24T04:30:51,081 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35619-0x101483d427d0001, quorum=127.0.0.1:51577, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-24T04:30:51,081 INFO  [RS:0;fadb2fe0ba32:35619 {}] regionserver.HRegionServer(1031): Exiting; stopping=fadb2fe0ba32,35619,1732422621246; zookeeper connection closed.
2024-11-24T04:30:51,082 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1ffe23d1 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1ffe23d1
2024-11-24T04:30:51,082 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-24T04:30:51,087 DEBUG [M:0;fadb2fe0ba32:39473 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/da9683c6378b4e219d619beda2cfed0e is 52, key is load_balancer_on/state:d/1732422622178/Put/seqid=0
2024-11-24T04:30:51,092 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34945 is added to blk_1073741851_1036 (size=5056)
2024-11-24T04:30:51,092 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35025 is added to blk_1073741851_1036 (size=5056)
2024-11-24T04:30:51,092 INFO  [M:0;fadb2fe0ba32:39473 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/da9683c6378b4e219d619beda2cfed0e
2024-11-24T04:30:51,097 DEBUG [M:0;fadb2fe0ba32:39473 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1392738e728d40418ac2cdbeaf242243 as hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1392738e728d40418ac2cdbeaf242243
2024-11-24T04:30:51,102 INFO  [M:0;fadb2fe0ba32:39473 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1392738e728d40418ac2cdbeaf242243, entries=8, sequenceid=56, filesize=5.5 K
2024-11-24T04:30:51,102 DEBUG [M:0;fadb2fe0ba32:39473 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d327da8dff91450c9c5affc6b52f18e5 as hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/d327da8dff91450c9c5affc6b52f18e5
2024-11-24T04:30:51,107 INFO  [M:0;fadb2fe0ba32:39473 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/d327da8dff91450c9c5affc6b52f18e5, entries=6, sequenceid=56, filesize=6.0 K
2024-11-24T04:30:51,108 DEBUG [M:0;fadb2fe0ba32:39473 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5a74581c64474ce58b5fb0354ace3be3 as hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/5a74581c64474ce58b5fb0354ace3be3
2024-11-24T04:30:51,113 INFO  [M:0;fadb2fe0ba32:39473 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/5a74581c64474ce58b5fb0354ace3be3, entries=1, sequenceid=56, filesize=5.0 K
2024-11-24T04:30:51,114 DEBUG [M:0;fadb2fe0ba32:39473 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/da9683c6378b4e219d619beda2cfed0e as hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/da9683c6378b4e219d619beda2cfed0e
2024-11-24T04:30:51,119 INFO  [M:0;fadb2fe0ba32:39473 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/da9683c6378b4e219d619beda2cfed0e, entries=1, sequenceid=56, filesize=4.9 K
2024-11-24T04:30:51,120 INFO  [M:0;fadb2fe0ba32:39473 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.17 KB/23726, heapSize ~29.10 KB/29800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 127ms, sequenceid=56, compaction requested=false
2024-11-24T04:30:51,121 INFO  [M:0;fadb2fe0ba32:39473 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T04:30:51,122 DEBUG [M:0;fadb2fe0ba32:39473 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732422650992Disabling compacts and flushes for region at 1732422650992Disabling writes for close at 1732422650993 (+1 ms)Obtaining lock to block concurrent updates at 1732422650993Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732422650993Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23726, getHeapSize=29800, getOffHeapSize=0, getCellsCount=67 at 1732422650993Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732422651000 (+7 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732422651000Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732422651015 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732422651015Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732422651025 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732422651039 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732422651039Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732422651050 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732422651063 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732422651063Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732422651073 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732422651087 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732422651087Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@78676127: reopening flushed file at 1732422651097 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@72879fc0: reopening flushed file at 1732422651102 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5cb394e4: reopening flushed file at 1732422651107 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6569d9c7: reopening flushed file at 1732422651113 (+6 ms)Finished flush of dataSize ~23.17 KB/23726, heapSize ~29.10 KB/29800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 127ms, sequenceid=56, compaction requested=false at 1732422651120 (+7 ms)Writing region close event to WAL at 1732422651121 (+1 ms)Closed at 1732422651121
2024-11-24T04:30:51,122 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:51,122 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:51,122 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:51,122 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:51,122 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:30:51,124 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34945 is added to blk_1073741847_1031 (size=757)
2024-11-24T04:30:51,124 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35025 is added to blk_1073741847_1031 (size=757)
2024-11-24T04:30:51,163 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-24T04:30:51,808 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:51,816 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:52,539 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:30:52,540 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:30:52,558 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:30:52,559 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:30:52,559 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:30:52,560 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:30:52,560 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:30:52,560 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:30:52,565 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:30:52,565 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:30:52,565 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:30:52,567 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:30:52,570 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:30:52,570 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:30:52,808 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:52,816 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:53,072 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-24T04:30:53,074 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:30:53,074 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:30:53,074 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:30:53,074 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:30:53,087 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:30:53,087 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:30:53,087 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:30:53,087 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:30:53,088 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:30:53,088 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:30:53,090 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:30:53,091 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:30:53,091 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:30:53,093 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:30:53,809 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:53,817 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:53,977 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741830_1015: GenerationStamp not matched, existing replica is blk_1073741830_1006
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-11-24T04:30:54,809 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:54,817 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:55,000 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/MasterData/WALs/fadb2fe0ba32,39473,1732422621180/fadb2fe0ba32%2C39473%2C1732422621180.1732422621341 after 4001ms
2024-11-24T04:30:55,001 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/MasterData/WALs/fadb2fe0ba32,39473,1732422621180/fadb2fe0ba32%2C39473%2C1732422621180.1732422621341 to hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/MasterData/oldWALs/fadb2fe0ba32%2C39473%2C1732422621180.1732422621341
2024-11-24T04:30:55,004 INFO  [WAL-Archive-0 {}] region.MasterRegionUtils(50): Moved hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/MasterData/oldWALs/fadb2fe0ba32%2C39473%2C1732422621180.1732422621341 to hdfs://localhost:38691/user/jenkins/test-data/7dc30bd7-a4c7-0917-d262-e91181f47ecf/oldWALs/fadb2fe0ba32%2C39473%2C1732422621180.1732422621341$masterlocalwal$
2024-11-24T04:30:55,004 INFO  [M:0;fadb2fe0ba32:39473 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-24T04:30:55,004 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-24T04:30:55,004 INFO  [M:0;fadb2fe0ba32:39473 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39473
2024-11-24T04:30:55,005 INFO  [M:0;fadb2fe0ba32:39473 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-24T04:30:55,106 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39473-0x101483d427d0000, quorum=127.0.0.1:51577, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-24T04:30:55,106 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39473-0x101483d427d0000, quorum=127.0.0.1:51577, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-24T04:30:55,106 INFO  [M:0;fadb2fe0ba32:39473 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-24T04:30:55,109 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3172a7c6{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T04:30:55,109 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7b559376{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-24T04:30:55,109 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-24T04:30:55,109 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@20466b6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-24T04:30:55,109 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@64685bd7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f4065fa-2df8-4567-fbb0-481a3f88ce22/hadoop.log.dir/,STOPPED}
2024-11-24T04:30:55,110 WARN  [BP-878497079-172.17.0.2-1732422620375 heartbeating to localhost/127.0.0.1:38691 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-24T04:30:55,110 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-24T04:30:55,110 WARN  [BP-878497079-172.17.0.2-1732422620375 heartbeating to localhost/127.0.0.1:38691 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-878497079-172.17.0.2-1732422620375 (Datanode Uuid 8339f64a-a65e-4a7c-aeab-6bb7febc774a) service to localhost/127.0.0.1:38691
2024-11-24T04:30:55,110 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-24T04:30:55,111 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f4065fa-2df8-4567-fbb0-481a3f88ce22/cluster_7730fbad-df8e-ea9e-97b3-261c7f308fa8/data/data3/current/BP-878497079-172.17.0.2-1732422620375 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T04:30:55,111 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f4065fa-2df8-4567-fbb0-481a3f88ce22/cluster_7730fbad-df8e-ea9e-97b3-261c7f308fa8/data/data4/current/BP-878497079-172.17.0.2-1732422620375 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T04:30:55,111 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-24T04:30:55,113 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@290ac13e{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T04:30:55,114 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@117b424d{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-24T04:30:55,114 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-24T04:30:55,114 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@fa662a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-24T04:30:55,114 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@57d5f4b3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f4065fa-2df8-4567-fbb0-481a3f88ce22/hadoop.log.dir/,STOPPED}
2024-11-24T04:30:55,115 WARN  [BP-878497079-172.17.0.2-1732422620375 heartbeating to localhost/127.0.0.1:38691 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-24T04:30:55,115 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-24T04:30:55,115 WARN  [BP-878497079-172.17.0.2-1732422620375 heartbeating to localhost/127.0.0.1:38691 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-878497079-172.17.0.2-1732422620375 (Datanode Uuid 5e400a8b-e68b-4fbe-8406-f707b7cad460) service to localhost/127.0.0.1:38691
2024-11-24T04:30:55,115 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-24T04:30:55,115 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f4065fa-2df8-4567-fbb0-481a3f88ce22/cluster_7730fbad-df8e-ea9e-97b3-261c7f308fa8/data/data1/current/BP-878497079-172.17.0.2-1732422620375 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T04:30:55,116 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f4065fa-2df8-4567-fbb0-481a3f88ce22/cluster_7730fbad-df8e-ea9e-97b3-261c7f308fa8/data/data2/current/BP-878497079-172.17.0.2-1732422620375 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T04:30:55,116 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-24T04:30:55,121 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5d7c8143{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-24T04:30:55,122 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6de1af65{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-24T04:30:55,122 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-24T04:30:55,122 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@17b2a9ba{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-24T04:30:55,122 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2ddfefdc{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f4065fa-2df8-4567-fbb0-481a3f88ce22/hadoop.log.dir/,STOPPED}
2024-11-24T04:30:55,128 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-24T04:30:55,145 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-24T04:30:55,153 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=179 (was 153)
Potentially hanging thread: nioEventLoopGroup-31-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:38691 from jenkins.hfs.4
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: HMaster-EventLoopGroup-10-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-33-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-31-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Timer for 'DataNode' metrics system
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: HMaster-EventLoopGroup-10-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-10-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-32-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.4@localhost:38691
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-33-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:38691
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-30-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-32-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-30-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:38691 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:38691 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:38691
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-30-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-32-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:38691
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-33-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:38691
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-31-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=457 (was 450) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=103 (was 167), ProcessCount=11 (was 11), AvailableMemoryMB=6511 (was 6779)
2024-11-24T04:30:55,160 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=179, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=103, ProcessCount=11, AvailableMemoryMB=6511
2024-11-24T04:30:55,160 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-24T04:30:55,160 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f4065fa-2df8-4567-fbb0-481a3f88ce22/hadoop.log.dir so I do NOT create it in target/test-data/b4927217-95d0-6eee-a4fe-4f3752eebc02
2024-11-24T04:30:55,160 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7f4065fa-2df8-4567-fbb0-481a3f88ce22/hadoop.tmp.dir so I do NOT create it in target/test-data/b4927217-95d0-6eee-a4fe-4f3752eebc02
2024-11-24T04:30:55,160 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b4927217-95d0-6eee-a4fe-4f3752eebc02/cluster_ff778f8a-035a-c4d3-e51d-256e45f57157, deleteOnExit=true
2024-11-24T04:30:55,160 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-24T04:30:55,160 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b4927217-95d0-6eee-a4fe-4f3752eebc02/test.cache.data in system properties and HBase conf
2024-11-24T04:30:55,160 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b4927217-95d0-6eee-a4fe-4f3752eebc02/hadoop.tmp.dir in system properties and HBase conf
2024-11-24T04:30:55,160 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b4927217-95d0-6eee-a4fe-4f3752eebc02/hadoop.log.dir in system properties and HBase conf
2024-11-24T04:30:55,161 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b4927217-95d0-6eee-a4fe-4f3752eebc02/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-24T04:30:55,161 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b4927217-95d0-6eee-a4fe-4f3752eebc02/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-24T04:30:55,161 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-24T04:30:55,161 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-24T04:30:55,161 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b4927217-95d0-6eee-a4fe-4f3752eebc02/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-24T04:30:55,161 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b4927217-95d0-6eee-a4fe-4f3752eebc02/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-24T04:30:55,161 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b4927217-95d0-6eee-a4fe-4f3752eebc02/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-24T04:30:55,161 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b4927217-95d0-6eee-a4fe-4f3752eebc02/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-24T04:30:55,161 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b4927217-95d0-6eee-a4fe-4f3752eebc02/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-24T04:30:55,161 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b4927217-95d0-6eee-a4fe-4f3752eebc02/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-24T04:30:55,161 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b4927217-95d0-6eee-a4fe-4f3752eebc02/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-24T04:30:55,161 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b4927217-95d0-6eee-a4fe-4f3752eebc02/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-24T04:30:55,161 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b4927217-95d0-6eee-a4fe-4f3752eebc02/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-24T04:30:55,161 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b4927217-95d0-6eee-a4fe-4f3752eebc02/nfs.dump.dir in system properties and HBase conf
2024-11-24T04:30:55,161 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b4927217-95d0-6eee-a4fe-4f3752eebc02/java.io.tmpdir in system properties and HBase conf
2024-11-24T04:30:55,161 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b4927217-95d0-6eee-a4fe-4f3752eebc02/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-24T04:30:55,162 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b4927217-95d0-6eee-a4fe-4f3752eebc02/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-24T04:30:55,162 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b4927217-95d0-6eee-a4fe-4f3752eebc02/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-24T04:30:55,175 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-24T04:30:55,240 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T04:30:55,243 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-24T04:30:55,245 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-24T04:30:55,245 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-24T04:30:55,245 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-24T04:30:55,245 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T04:30:55,246 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@70e475c9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b4927217-95d0-6eee-a4fe-4f3752eebc02/hadoop.log.dir/,AVAILABLE}
2024-11-24T04:30:55,246 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@290f188f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-24T04:30:55,360 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@62558ec9{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b4927217-95d0-6eee-a4fe-4f3752eebc02/java.io.tmpdir/jetty-localhost-32835-hadoop-hdfs-3_4_1-tests_jar-_-any-8545659437127114566/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-24T04:30:55,361 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@294894f5{HTTP/1.1, (http/1.1)}{localhost:32835}
2024-11-24T04:30:55,361 INFO  [Time-limited test {}] server.Server(415): Started @184422ms
2024-11-24T04:30:55,374 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-24T04:30:55,428 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T04:30:55,431 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-24T04:30:55,432 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-24T04:30:55,432 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-24T04:30:55,432 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-24T04:30:55,432 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2e6ee234{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b4927217-95d0-6eee-a4fe-4f3752eebc02/hadoop.log.dir/,AVAILABLE}
2024-11-24T04:30:55,433 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5b5b6aa1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-24T04:30:55,549 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@48bfafbe{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b4927217-95d0-6eee-a4fe-4f3752eebc02/java.io.tmpdir/jetty-localhost-38211-hadoop-hdfs-3_4_1-tests_jar-_-any-7971523387165376667/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T04:30:55,550 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2b93cee9{HTTP/1.1, (http/1.1)}{localhost:38211}
2024-11-24T04:30:55,550 INFO  [Time-limited test {}] server.Server(415): Started @184611ms
2024-11-24T04:30:55,551 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-24T04:30:55,579 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T04:30:55,582 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-24T04:30:55,583 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-24T04:30:55,583 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-24T04:30:55,583 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-24T04:30:55,583 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@20f59884{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b4927217-95d0-6eee-a4fe-4f3752eebc02/hadoop.log.dir/,AVAILABLE}
2024-11-24T04:30:55,583 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@190ad9e7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-24T04:30:55,653 WARN  [Thread-1639 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b4927217-95d0-6eee-a4fe-4f3752eebc02/cluster_ff778f8a-035a-c4d3-e51d-256e45f57157/data/data1/current/BP-946574629-172.17.0.2-1732422655191/current, will proceed with Du for space computation calculation, 
2024-11-24T04:30:55,654 WARN  [Thread-1640 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b4927217-95d0-6eee-a4fe-4f3752eebc02/cluster_ff778f8a-035a-c4d3-e51d-256e45f57157/data/data2/current/BP-946574629-172.17.0.2-1732422655191/current, will proceed with Du for space computation calculation, 
2024-11-24T04:30:55,670 WARN  [Thread-1618 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-24T04:30:55,673 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa11772d621ad4342 with lease ID 0xfba4ee7b2b6ec6ea: Processing first storage report for DS-1be70c02-7ccb-4b7b-938a-79e6161e11ee from datanode DatanodeRegistration(127.0.0.1:36061, datanodeUuid=30392bac-e63a-48f8-a36f-d0105107d846, infoPort=42935, infoSecurePort=0, ipcPort=33039, storageInfo=lv=-57;cid=testClusterID;nsid=131910990;c=1732422655191)
2024-11-24T04:30:55,673 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa11772d621ad4342 with lease ID 0xfba4ee7b2b6ec6ea: from storage DS-1be70c02-7ccb-4b7b-938a-79e6161e11ee node DatanodeRegistration(127.0.0.1:36061, datanodeUuid=30392bac-e63a-48f8-a36f-d0105107d846, infoPort=42935, infoSecurePort=0, ipcPort=33039, storageInfo=lv=-57;cid=testClusterID;nsid=131910990;c=1732422655191), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T04:30:55,673 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa11772d621ad4342 with lease ID 0xfba4ee7b2b6ec6ea: Processing first storage report for DS-40c7ba7b-a73d-4959-b88e-620c708ba6c8 from datanode DatanodeRegistration(127.0.0.1:36061, datanodeUuid=30392bac-e63a-48f8-a36f-d0105107d846, infoPort=42935, infoSecurePort=0, ipcPort=33039, storageInfo=lv=-57;cid=testClusterID;nsid=131910990;c=1732422655191)
2024-11-24T04:30:55,673 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa11772d621ad4342 with lease ID 0xfba4ee7b2b6ec6ea: from storage DS-40c7ba7b-a73d-4959-b88e-620c708ba6c8 node DatanodeRegistration(127.0.0.1:36061, datanodeUuid=30392bac-e63a-48f8-a36f-d0105107d846, infoPort=42935, infoSecurePort=0, ipcPort=33039, storageInfo=lv=-57;cid=testClusterID;nsid=131910990;c=1732422655191), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T04:30:55,700 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@fc981fd{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b4927217-95d0-6eee-a4fe-4f3752eebc02/java.io.tmpdir/jetty-localhost-33899-hadoop-hdfs-3_4_1-tests_jar-_-any-4610197492064799371/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T04:30:55,701 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1367dc96{HTTP/1.1, (http/1.1)}{localhost:33899}
2024-11-24T04:30:55,701 INFO  [Time-limited test {}] server.Server(415): Started @184762ms
2024-11-24T04:30:55,702 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-24T04:30:55,810 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:55,818 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:55,818 WARN  [Thread-1665 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b4927217-95d0-6eee-a4fe-4f3752eebc02/cluster_ff778f8a-035a-c4d3-e51d-256e45f57157/data/data3/current/BP-946574629-172.17.0.2-1732422655191/current, will proceed with Du for space computation calculation, 
2024-11-24T04:30:55,818 WARN  [Thread-1666 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b4927217-95d0-6eee-a4fe-4f3752eebc02/cluster_ff778f8a-035a-c4d3-e51d-256e45f57157/data/data4/current/BP-946574629-172.17.0.2-1732422655191/current, will proceed with Du for space computation calculation, 
2024-11-24T04:30:55,837 WARN  [Thread-1654 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-24T04:30:55,839 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6f1df5dee9157573 with lease ID 0xfba4ee7b2b6ec6eb: Processing first storage report for DS-cf46102c-9013-4649-a8b8-ff8b92f55910 from datanode DatanodeRegistration(127.0.0.1:45239, datanodeUuid=10eef7a0-3f4e-4e45-91ad-1f87f08dea30, infoPort=42033, infoSecurePort=0, ipcPort=34807, storageInfo=lv=-57;cid=testClusterID;nsid=131910990;c=1732422655191)
2024-11-24T04:30:55,839 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6f1df5dee9157573 with lease ID 0xfba4ee7b2b6ec6eb: from storage DS-cf46102c-9013-4649-a8b8-ff8b92f55910 node DatanodeRegistration(127.0.0.1:45239, datanodeUuid=10eef7a0-3f4e-4e45-91ad-1f87f08dea30, infoPort=42033, infoSecurePort=0, ipcPort=34807, storageInfo=lv=-57;cid=testClusterID;nsid=131910990;c=1732422655191), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T04:30:55,839 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6f1df5dee9157573 with lease ID 0xfba4ee7b2b6ec6eb: Processing first storage report for DS-aa9a3912-6433-4d5a-81f6-867fbfe46c60 from datanode DatanodeRegistration(127.0.0.1:45239, datanodeUuid=10eef7a0-3f4e-4e45-91ad-1f87f08dea30, infoPort=42033, infoSecurePort=0, ipcPort=34807, storageInfo=lv=-57;cid=testClusterID;nsid=131910990;c=1732422655191)
2024-11-24T04:30:55,839 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6f1df5dee9157573 with lease ID 0xfba4ee7b2b6ec6eb: from storage DS-aa9a3912-6433-4d5a-81f6-867fbfe46c60 node DatanodeRegistration(127.0.0.1:45239, datanodeUuid=10eef7a0-3f4e-4e45-91ad-1f87f08dea30, infoPort=42033, infoSecurePort=0, ipcPort=34807, storageInfo=lv=-57;cid=testClusterID;nsid=131910990;c=1732422655191), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T04:30:55,926 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b4927217-95d0-6eee-a4fe-4f3752eebc02
2024-11-24T04:30:55,928 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b4927217-95d0-6eee-a4fe-4f3752eebc02/cluster_ff778f8a-035a-c4d3-e51d-256e45f57157/zookeeper_0, clientPort=49920, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b4927217-95d0-6eee-a4fe-4f3752eebc02/cluster_ff778f8a-035a-c4d3-e51d-256e45f57157/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b4927217-95d0-6eee-a4fe-4f3752eebc02/cluster_ff778f8a-035a-c4d3-e51d-256e45f57157/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-24T04:30:55,929 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=49920
2024-11-24T04:30:55,929 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T04:30:55,931 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T04:30:55,938 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36061 is added to blk_1073741825_1001 (size=7)
2024-11-24T04:30:55,939 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45239 is added to blk_1073741825_1001 (size=7)
2024-11-24T04:30:55,940 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28 with version=8
2024-11-24T04:30:55,940 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/hbase-staging
2024-11-24T04:30:55,942 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/fadb2fe0ba32:0 server-side Connection retries=45
2024-11-24T04:30:55,942 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-24T04:30:55,942 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-24T04:30:55,942 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-24T04:30:55,942 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-24T04:30:55,942 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-24T04:30:55,942 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-24T04:30:55,942 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-24T04:30:55,943 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35553
2024-11-24T04:30:55,944 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35553 connecting to ZooKeeper ensemble=127.0.0.1:49920
2024-11-24T04:30:55,951 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:355530x0, quorum=127.0.0.1:49920, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-24T04:30:55,952 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35553-0x101483dca470000 connected
2024-11-24T04:30:55,966 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T04:30:55,967 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T04:30:55,969 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35553-0x101483dca470000, quorum=127.0.0.1:49920, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-24T04:30:55,969 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28, hbase.cluster.distributed=false
2024-11-24T04:30:55,971 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35553-0x101483dca470000, quorum=127.0.0.1:49920, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-24T04:30:55,971 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35553
2024-11-24T04:30:55,971 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35553
2024-11-24T04:30:55,972 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35553
2024-11-24T04:30:55,972 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35553
2024-11-24T04:30:55,972 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35553
2024-11-24T04:30:55,988 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/fadb2fe0ba32:0 server-side Connection retries=45
2024-11-24T04:30:55,988 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-24T04:30:55,988 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-24T04:30:55,988 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-24T04:30:55,988 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-24T04:30:55,988 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-24T04:30:55,988 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-24T04:30:55,988 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-24T04:30:55,989 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41373
2024-11-24T04:30:55,990 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41373 connecting to ZooKeeper ensemble=127.0.0.1:49920
2024-11-24T04:30:55,991 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T04:30:55,992 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T04:30:56,002 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:413730x0, quorum=127.0.0.1:49920, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-24T04:30:56,003 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:413730x0, quorum=127.0.0.1:49920, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-24T04:30:56,003 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41373-0x101483dca470001 connected
2024-11-24T04:30:56,003 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-24T04:30:56,004 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-24T04:30:56,004 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41373-0x101483dca470001, quorum=127.0.0.1:49920, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-24T04:30:56,005 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41373-0x101483dca470001, quorum=127.0.0.1:49920, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-24T04:30:56,005 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41373
2024-11-24T04:30:56,006 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41373
2024-11-24T04:30:56,006 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41373
2024-11-24T04:30:56,006 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41373
2024-11-24T04:30:56,006 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41373
2024-11-24T04:30:56,019 DEBUG [M:0;fadb2fe0ba32:35553 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;fadb2fe0ba32:35553
2024-11-24T04:30:56,019 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/fadb2fe0ba32,35553,1732422655941
2024-11-24T04:30:56,020 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41373-0x101483dca470001, quorum=127.0.0.1:49920, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-24T04:30:56,020 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35553-0x101483dca470000, quorum=127.0.0.1:49920, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-24T04:30:56,021 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35553-0x101483dca470000, quorum=127.0.0.1:49920, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/fadb2fe0ba32,35553,1732422655941
2024-11-24T04:30:56,023 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41373-0x101483dca470001, quorum=127.0.0.1:49920, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-24T04:30:56,023 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35553-0x101483dca470000, quorum=127.0.0.1:49920, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:30:56,023 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41373-0x101483dca470001, quorum=127.0.0.1:49920, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:30:56,023 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35553-0x101483dca470000, quorum=127.0.0.1:49920, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-24T04:30:56,024 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/fadb2fe0ba32,35553,1732422655941 from backup master directory
2024-11-24T04:30:56,025 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35553-0x101483dca470000, quorum=127.0.0.1:49920, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/fadb2fe0ba32,35553,1732422655941
2024-11-24T04:30:56,025 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41373-0x101483dca470001, quorum=127.0.0.1:49920, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-24T04:30:56,025 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35553-0x101483dca470000, quorum=127.0.0.1:49920, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-24T04:30:56,025 WARN  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-24T04:30:56,025 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=fadb2fe0ba32,35553,1732422655941
2024-11-24T04:30:56,029 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/hbase.id] with ID: 5685d04a-4df7-45da-9ee3-6eb5b42fb61e
2024-11-24T04:30:56,029 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/.tmp/hbase.id
2024-11-24T04:30:56,035 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45239 is added to blk_1073741826_1002 (size=42)
2024-11-24T04:30:56,035 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36061 is added to blk_1073741826_1002 (size=42)
2024-11-24T04:30:56,036 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/.tmp/hbase.id]:[hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/hbase.id]
2024-11-24T04:30:56,046 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T04:30:56,046 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-24T04:30:56,048 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms.
2024-11-24T04:30:56,051 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41373-0x101483dca470001, quorum=127.0.0.1:49920, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:30:56,051 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35553-0x101483dca470000, quorum=127.0.0.1:49920, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:30:56,057 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36061 is added to blk_1073741827_1003 (size=196)
2024-11-24T04:30:56,058 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45239 is added to blk_1073741827_1003 (size=196)
2024-11-24T04:30:56,058 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-24T04:30:56,059 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-24T04:30:56,059 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-24T04:30:56,065 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45239 is added to blk_1073741828_1004 (size=1189)
2024-11-24T04:30:56,066 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36061 is added to blk_1073741828_1004 (size=1189)
2024-11-24T04:30:56,066 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/MasterData/data/master/store
2024-11-24T04:30:56,072 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36061 is added to blk_1073741829_1005 (size=34)
2024-11-24T04:30:56,072 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45239 is added to blk_1073741829_1005 (size=34)
2024-11-24T04:30:56,073 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T04:30:56,073 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-24T04:30:56,073 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T04:30:56,073 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T04:30:56,073 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-24T04:30:56,073 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T04:30:56,073 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T04:30:56,073 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732422656073Disabling compacts and flushes for region at 1732422656073Disabling writes for close at 1732422656073Writing region close event to WAL at 1732422656073Closed at 1732422656073
2024-11-24T04:30:56,074 WARN  [master/fadb2fe0ba32:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/MasterData/data/master/store/.initializing
2024-11-24T04:30:56,074 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/MasterData/WALs/fadb2fe0ba32,35553,1732422655941
2024-11-24T04:30:56,076 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fadb2fe0ba32%2C35553%2C1732422655941, suffix=, logDir=hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/MasterData/WALs/fadb2fe0ba32,35553,1732422655941, archiveDir=hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/MasterData/oldWALs, maxLogs=10
2024-11-24T04:30:56,076 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor fadb2fe0ba32%2C35553%2C1732422655941.1732422656076
2024-11-24T04:30:56,081 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/MasterData/WALs/fadb2fe0ba32,35553,1732422655941/fadb2fe0ba32%2C35553%2C1732422655941.1732422656076
2024-11-24T04:30:56,081 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42033:42033),(127.0.0.1/127.0.0.1:42935:42935)]
2024-11-24T04:30:56,082 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-24T04:30:56,082 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T04:30:56,082 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T04:30:56,082 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T04:30:56,083 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-24T04:30:56,085 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-24T04:30:56,085 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:30:56,085 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T04:30:56,085 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-24T04:30:56,086 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-24T04:30:56,086 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:30:56,087 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-24T04:30:56,087 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-24T04:30:56,088 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-24T04:30:56,088 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:30:56,088 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-24T04:30:56,088 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-24T04:30:56,089 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-24T04:30:56,089 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:30:56,089 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-24T04:30:56,090 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T04:30:56,090 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-24T04:30:56,090 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-24T04:30:56,091 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T04:30:56,092 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T04:30:56,092 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-24T04:30:56,093 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T04:30:56,095 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-24T04:30:56,095 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=785552, jitterRate=-0.001120239496231079}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-24T04:30:56,096 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1732422656083Initializing all the Stores at 1732422656083Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732422656083Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422656083Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422656083Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422656083Cleaning up temporary data from old regions at 1732422656092 (+9 ms)Region opened successfully at 1732422656096 (+4 ms)
2024-11-24T04:30:56,096 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-24T04:30:56,099 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3de023b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fadb2fe0ba32/172.17.0.2:0
2024-11-24T04:30:56,100 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-24T04:30:56,101 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-24T04:30:56,101 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-24T04:30:56,101 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-24T04:30:56,101 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-24T04:30:56,101 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-24T04:30:56,101 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-24T04:30:56,103 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-24T04:30:56,104 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35553-0x101483dca470000, quorum=127.0.0.1:49920, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-24T04:30:56,105 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-24T04:30:56,106 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-24T04:30:56,106 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35553-0x101483dca470000, quorum=127.0.0.1:49920, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-24T04:30:56,107 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-24T04:30:56,108 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-24T04:30:56,108 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35553-0x101483dca470000, quorum=127.0.0.1:49920, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-24T04:30:56,110 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-24T04:30:56,110 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35553-0x101483dca470000, quorum=127.0.0.1:49920, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-24T04:30:56,113 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-24T04:30:56,115 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35553-0x101483dca470000, quorum=127.0.0.1:49920, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-24T04:30:56,116 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-24T04:30:56,118 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35553-0x101483dca470000, quorum=127.0.0.1:49920, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-24T04:30:56,118 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41373-0x101483dca470001, quorum=127.0.0.1:49920, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-24T04:30:56,118 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35553-0x101483dca470000, quorum=127.0.0.1:49920, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:30:56,118 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41373-0x101483dca470001, quorum=127.0.0.1:49920, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:30:56,118 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=fadb2fe0ba32,35553,1732422655941, sessionid=0x101483dca470000, setting cluster-up flag (Was=false)
2024-11-24T04:30:56,122 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35553-0x101483dca470000, quorum=127.0.0.1:49920, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:30:56,122 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41373-0x101483dca470001, quorum=127.0.0.1:49920, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:30:56,127 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-24T04:30:56,127 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=fadb2fe0ba32,35553,1732422655941
2024-11-24T04:30:56,132 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35553-0x101483dca470000, quorum=127.0.0.1:49920, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:30:56,132 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41373-0x101483dca470001, quorum=127.0.0.1:49920, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:30:56,137 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-24T04:30:56,138 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=fadb2fe0ba32,35553,1732422655941
2024-11-24T04:30:56,139 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-24T04:30:56,141 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-24T04:30:56,141 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-24T04:30:56,141 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-24T04:30:56,141 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: fadb2fe0ba32,35553,1732422655941
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-24T04:30:56,142 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/fadb2fe0ba32:0, corePoolSize=5, maxPoolSize=5
2024-11-24T04:30:56,142 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/fadb2fe0ba32:0, corePoolSize=5, maxPoolSize=5
2024-11-24T04:30:56,143 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/fadb2fe0ba32:0, corePoolSize=5, maxPoolSize=5
2024-11-24T04:30:56,143 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/fadb2fe0ba32:0, corePoolSize=5, maxPoolSize=5
2024-11-24T04:30:56,143 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/fadb2fe0ba32:0, corePoolSize=10, maxPoolSize=10
2024-11-24T04:30:56,143 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:30:56,143 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/fadb2fe0ba32:0, corePoolSize=2, maxPoolSize=2
2024-11-24T04:30:56,143 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:30:56,144 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732422686144
2024-11-24T04:30:56,144 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-24T04:30:56,144 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-24T04:30:56,144 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-24T04:30:56,144 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-24T04:30:56,144 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-24T04:30:56,144 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-24T04:30:56,145 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-24T04:30:56,145 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-24T04:30:56,145 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-24T04:30:56,145 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-24T04:30:56,145 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-24T04:30:56,145 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-24T04:30:56,145 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-24T04:30:56,146 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-24T04:30:56,146 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/fadb2fe0ba32:0:becomeActiveMaster-HFileCleaner.large.0-1732422656146,5,FailOnTimeoutGroup]
2024-11-24T04:30:56,146 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:30:56,146 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/fadb2fe0ba32:0:becomeActiveMaster-HFileCleaner.small.0-1732422656146,5,FailOnTimeoutGroup]
2024-11-24T04:30:56,146 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-24T04:30:56,146 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-24T04:30:56,146 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-24T04:30:56,146 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-24T04:30:56,146 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-24T04:30:56,153 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36061 is added to blk_1073741831_1007 (size=1321)
2024-11-24T04:30:56,153 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45239 is added to blk_1073741831_1007 (size=1321)
2024-11-24T04:30:56,154 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-24T04:30:56,154 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28
2024-11-24T04:30:56,160 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45239 is added to blk_1073741832_1008 (size=32)
2024-11-24T04:30:56,161 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36061 is added to blk_1073741832_1008 (size=32)
2024-11-24T04:30:56,162 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T04:30:56,163 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-24T04:30:56,164 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-24T04:30:56,164 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:30:56,165 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T04:30:56,165 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-24T04:30:56,166 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-24T04:30:56,166 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:30:56,166 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T04:30:56,167 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-24T04:30:56,168 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-24T04:30:56,168 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:30:56,168 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T04:30:56,168 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-24T04:30:56,169 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-24T04:30:56,169 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:30:56,170 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T04:30:56,170 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-24T04:30:56,170 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/data/hbase/meta/1588230740
2024-11-24T04:30:56,171 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/data/hbase/meta/1588230740
2024-11-24T04:30:56,172 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-24T04:30:56,172 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-24T04:30:56,172 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-24T04:30:56,173 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-24T04:30:56,175 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-24T04:30:56,176 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=752932, jitterRate=-0.04259781539440155}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-24T04:30:56,176 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1732422656162Initializing all the Stores at 1732422656163 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732422656163Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732422656163Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422656163Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732422656163Cleaning up temporary data from old regions at 1732422656172 (+9 ms)Region opened successfully at 1732422656176 (+4 ms)
2024-11-24T04:30:56,177 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-24T04:30:56,177 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-24T04:30:56,177 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-24T04:30:56,177 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-24T04:30:56,177 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-24T04:30:56,177 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-24T04:30:56,177 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732422656177Disabling compacts and flushes for region at 1732422656177Disabling writes for close at 1732422656177Writing region close event to WAL at 1732422656177Closed at 1732422656177
2024-11-24T04:30:56,178 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-24T04:30:56,178 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-24T04:30:56,178 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-24T04:30:56,180 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-24T04:30:56,181 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-24T04:30:56,208 INFO  [RS:0;fadb2fe0ba32:41373 {}] regionserver.HRegionServer(746): ClusterId : 5685d04a-4df7-45da-9ee3-6eb5b42fb61e
2024-11-24T04:30:56,208 DEBUG [RS:0;fadb2fe0ba32:41373 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-24T04:30:56,211 DEBUG [RS:0;fadb2fe0ba32:41373 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-24T04:30:56,211 DEBUG [RS:0;fadb2fe0ba32:41373 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-24T04:30:56,212 DEBUG [RS:0;fadb2fe0ba32:41373 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-24T04:30:56,213 DEBUG [RS:0;fadb2fe0ba32:41373 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@365dadc3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fadb2fe0ba32/172.17.0.2:0
2024-11-24T04:30:56,225 DEBUG [RS:0;fadb2fe0ba32:41373 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;fadb2fe0ba32:41373
2024-11-24T04:30:56,225 INFO  [RS:0;fadb2fe0ba32:41373 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-24T04:30:56,225 INFO  [RS:0;fadb2fe0ba32:41373 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-24T04:30:56,225 DEBUG [RS:0;fadb2fe0ba32:41373 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-24T04:30:56,226 INFO  [RS:0;fadb2fe0ba32:41373 {}] regionserver.HRegionServer(2659): reportForDuty to master=fadb2fe0ba32,35553,1732422655941 with port=41373, startcode=1732422655987
2024-11-24T04:30:56,226 DEBUG [RS:0;fadb2fe0ba32:41373 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-24T04:30:56,228 INFO  [HMaster-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59543, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.5 (auth:SIMPLE), service=RegionServerStatusService
2024-11-24T04:30:56,229 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35553 {}] master.ServerManager(363): Checking decommissioned status of RegionServer fadb2fe0ba32,41373,1732422655987
2024-11-24T04:30:56,229 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35553 {}] master.ServerManager(517): Registering regionserver=fadb2fe0ba32,41373,1732422655987
2024-11-24T04:30:56,231 DEBUG [RS:0;fadb2fe0ba32:41373 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28
2024-11-24T04:30:56,231 DEBUG [RS:0;fadb2fe0ba32:41373 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35273
2024-11-24T04:30:56,231 DEBUG [RS:0;fadb2fe0ba32:41373 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-24T04:30:56,232 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35553-0x101483dca470000, quorum=127.0.0.1:49920, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-24T04:30:56,233 DEBUG [RS:0;fadb2fe0ba32:41373 {}] zookeeper.ZKUtil(111): regionserver:41373-0x101483dca470001, quorum=127.0.0.1:49920, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/fadb2fe0ba32,41373,1732422655987
2024-11-24T04:30:56,233 WARN  [RS:0;fadb2fe0ba32:41373 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-24T04:30:56,233 INFO  [RS:0;fadb2fe0ba32:41373 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-24T04:30:56,233 DEBUG [RS:0;fadb2fe0ba32:41373 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/WALs/fadb2fe0ba32,41373,1732422655987
2024-11-24T04:30:56,233 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [fadb2fe0ba32,41373,1732422655987]
2024-11-24T04:30:56,236 INFO  [RS:0;fadb2fe0ba32:41373 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-24T04:30:56,238 INFO  [RS:0;fadb2fe0ba32:41373 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-24T04:30:56,238 INFO  [RS:0;fadb2fe0ba32:41373 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-24T04:30:56,238 INFO  [RS:0;fadb2fe0ba32:41373 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-24T04:30:56,238 INFO  [RS:0;fadb2fe0ba32:41373 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-24T04:30:56,239 INFO  [RS:0;fadb2fe0ba32:41373 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-24T04:30:56,239 INFO  [RS:0;fadb2fe0ba32:41373 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-24T04:30:56,239 DEBUG [RS:0;fadb2fe0ba32:41373 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:30:56,239 DEBUG [RS:0;fadb2fe0ba32:41373 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:30:56,239 DEBUG [RS:0;fadb2fe0ba32:41373 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:30:56,239 DEBUG [RS:0;fadb2fe0ba32:41373 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:30:56,239 DEBUG [RS:0;fadb2fe0ba32:41373 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:30:56,239 DEBUG [RS:0;fadb2fe0ba32:41373 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/fadb2fe0ba32:0, corePoolSize=2, maxPoolSize=2
2024-11-24T04:30:56,239 DEBUG [RS:0;fadb2fe0ba32:41373 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:30:56,239 DEBUG [RS:0;fadb2fe0ba32:41373 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:30:56,240 DEBUG [RS:0;fadb2fe0ba32:41373 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:30:56,240 DEBUG [RS:0;fadb2fe0ba32:41373 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:30:56,240 DEBUG [RS:0;fadb2fe0ba32:41373 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:30:56,240 DEBUG [RS:0;fadb2fe0ba32:41373 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:30:56,240 DEBUG [RS:0;fadb2fe0ba32:41373 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/fadb2fe0ba32:0, corePoolSize=3, maxPoolSize=3
2024-11-24T04:30:56,240 DEBUG [RS:0;fadb2fe0ba32:41373 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/fadb2fe0ba32:0, corePoolSize=3, maxPoolSize=3
2024-11-24T04:30:56,240 INFO  [RS:0;fadb2fe0ba32:41373 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-24T04:30:56,240 INFO  [RS:0;fadb2fe0ba32:41373 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-24T04:30:56,240 INFO  [RS:0;fadb2fe0ba32:41373 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-24T04:30:56,240 INFO  [RS:0;fadb2fe0ba32:41373 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-24T04:30:56,241 INFO  [RS:0;fadb2fe0ba32:41373 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-24T04:30:56,241 INFO  [RS:0;fadb2fe0ba32:41373 {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,41373,1732422655987-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-24T04:30:56,257 INFO  [RS:0;fadb2fe0ba32:41373 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-24T04:30:56,258 INFO  [RS:0;fadb2fe0ba32:41373 {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,41373,1732422655987-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-24T04:30:56,258 INFO  [RS:0;fadb2fe0ba32:41373 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T04:30:56,258 INFO  [RS:0;fadb2fe0ba32:41373 {}] regionserver.Replication(171): fadb2fe0ba32,41373,1732422655987 started
2024-11-24T04:30:56,266 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-24T04:30:56,266 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-24T04:30:56,266 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart
2024-11-24T04:30:56,272 INFO  [RS:0;fadb2fe0ba32:41373 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T04:30:56,272 INFO  [RS:0;fadb2fe0ba32:41373 {}] regionserver.HRegionServer(1482): Serving as fadb2fe0ba32,41373,1732422655987, RpcServer on fadb2fe0ba32/172.17.0.2:41373, sessionid=0x101483dca470001
2024-11-24T04:30:56,272 DEBUG [RS:0;fadb2fe0ba32:41373 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-24T04:30:56,272 DEBUG [RS:0;fadb2fe0ba32:41373 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager fadb2fe0ba32,41373,1732422655987
2024-11-24T04:30:56,272 DEBUG [RS:0;fadb2fe0ba32:41373 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fadb2fe0ba32,41373,1732422655987'
2024-11-24T04:30:56,272 DEBUG [RS:0;fadb2fe0ba32:41373 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-24T04:30:56,273 DEBUG [RS:0;fadb2fe0ba32:41373 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-24T04:30:56,274 DEBUG [RS:0;fadb2fe0ba32:41373 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-24T04:30:56,274 DEBUG [RS:0;fadb2fe0ba32:41373 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-24T04:30:56,274 DEBUG [RS:0;fadb2fe0ba32:41373 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager fadb2fe0ba32,41373,1732422655987
2024-11-24T04:30:56,274 DEBUG [RS:0;fadb2fe0ba32:41373 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fadb2fe0ba32,41373,1732422655987'
2024-11-24T04:30:56,274 DEBUG [RS:0;fadb2fe0ba32:41373 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-24T04:30:56,274 DEBUG [RS:0;fadb2fe0ba32:41373 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-24T04:30:56,274 DEBUG [RS:0;fadb2fe0ba32:41373 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-24T04:30:56,274 INFO  [RS:0;fadb2fe0ba32:41373 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-24T04:30:56,274 INFO  [RS:0;fadb2fe0ba32:41373 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-24T04:30:56,331 WARN  [fadb2fe0ba32:35553 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-11-24T04:30:56,376 INFO  [RS:0;fadb2fe0ba32:41373 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fadb2fe0ba32%2C41373%2C1732422655987, suffix=, logDir=hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/WALs/fadb2fe0ba32,41373,1732422655987, archiveDir=hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/oldWALs, maxLogs=32
2024-11-24T04:30:56,377 INFO  [RS:0;fadb2fe0ba32:41373 {}] monitor.StreamSlowMonitor(122): New stream slow monitor fadb2fe0ba32%2C41373%2C1732422655987.1732422656377
2024-11-24T04:30:56,382 INFO  [RS:0;fadb2fe0ba32:41373 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/WALs/fadb2fe0ba32,41373,1732422655987/fadb2fe0ba32%2C41373%2C1732422655987.1732422656377
2024-11-24T04:30:56,383 DEBUG [RS:0;fadb2fe0ba32:41373 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42033:42033),(127.0.0.1/127.0.0.1:42935:42935)]
2024-11-24T04:30:56,581 DEBUG [fadb2fe0ba32:35553 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-24T04:30:56,582 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=fadb2fe0ba32,41373,1732422655987
2024-11-24T04:30:56,583 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fadb2fe0ba32,41373,1732422655987, state=OPENING
2024-11-24T04:30:56,586 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-24T04:30:56,588 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41373-0x101483dca470001, quorum=127.0.0.1:49920, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:30:56,588 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35553-0x101483dca470000, quorum=127.0.0.1:49920, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:30:56,589 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-24T04:30:56,589 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=fadb2fe0ba32,41373,1732422655987}]
2024-11-24T04:30:56,589 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-24T04:30:56,589 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-24T04:30:56,743 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-24T04:30:56,745 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37239, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-24T04:30:56,748 INFO  [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-24T04:30:56,748 INFO  [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-24T04:30:56,750 INFO  [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fadb2fe0ba32%2C41373%2C1732422655987.meta, suffix=.meta, logDir=hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/WALs/fadb2fe0ba32,41373,1732422655987, archiveDir=hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/oldWALs, maxLogs=32
2024-11-24T04:30:56,750 INFO  [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor fadb2fe0ba32%2C41373%2C1732422655987.meta.1732422656750.meta
2024-11-24T04:30:56,758 INFO  [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/WALs/fadb2fe0ba32,41373,1732422655987/fadb2fe0ba32%2C41373%2C1732422655987.meta.1732422656750.meta
2024-11-24T04:30:56,760 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42033:42033),(127.0.0.1/127.0.0.1:42935:42935)]
2024-11-24T04:30:56,764 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-24T04:30:56,765 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-24T04:30:56,765 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-24T04:30:56,765 INFO  [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-24T04:30:56,765 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-24T04:30:56,765 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T04:30:56,765 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-24T04:30:56,766 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-24T04:30:56,767 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-24T04:30:56,768 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-24T04:30:56,768 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:30:56,768 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T04:30:56,769 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-24T04:30:56,769 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-24T04:30:56,769 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:30:56,770 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T04:30:56,770 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-24T04:30:56,770 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-24T04:30:56,770 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:30:56,771 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T04:30:56,771 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-24T04:30:56,771 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-24T04:30:56,771 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:30:56,772 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T04:30:56,772 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-24T04:30:56,772 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/data/hbase/meta/1588230740
2024-11-24T04:30:56,773 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/data/hbase/meta/1588230740
2024-11-24T04:30:56,774 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-24T04:30:56,774 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-24T04:30:56,775 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-24T04:30:56,775 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-24T04:30:56,776 INFO  [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=822547, jitterRate=0.045922815799713135}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-24T04:30:56,776 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-24T04:30:56,777 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1732422656766Writing region info on filesystem at 1732422656766Initializing all the Stores at 1732422656767 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732422656767Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732422656767Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422656767Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732422656767Cleaning up temporary data from old regions at 1732422656774 (+7 ms)Running coprocessor post-open hooks at 1732422656776 (+2 ms)Region opened successfully at 1732422656777 (+1 ms)
2024-11-24T04:30:56,778 INFO  [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732422656742
2024-11-24T04:30:56,781 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-24T04:30:56,781 INFO  [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-24T04:30:56,782 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=fadb2fe0ba32,41373,1732422655987
2024-11-24T04:30:56,783 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fadb2fe0ba32,41373,1732422655987, state=OPEN
2024-11-24T04:30:56,787 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41373-0x101483dca470001, quorum=127.0.0.1:49920, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-24T04:30:56,787 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35553-0x101483dca470000, quorum=127.0.0.1:49920, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-24T04:30:56,787 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=fadb2fe0ba32,41373,1732422655987
2024-11-24T04:30:56,787 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-24T04:30:56,787 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-24T04:30:56,790 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-24T04:30:56,790 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fadb2fe0ba32,41373,1732422655987 in 198 msec
2024-11-24T04:30:56,793 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-24T04:30:56,793 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 612 msec
2024-11-24T04:30:56,794 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-24T04:30:56,794 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-24T04:30:56,795 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-24T04:30:56,795 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fadb2fe0ba32,41373,1732422655987, seqNum=-1]
2024-11-24T04:30:56,795 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-24T04:30:56,797 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-13-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59769, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-24T04:30:56,802 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 661 msec
2024-11-24T04:30:56,802 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732422656802, completionTime=-1
2024-11-24T04:30:56,802 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-24T04:30:56,802 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-24T04:30:56,804 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-24T04:30:56,804 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732422716804
2024-11-24T04:30:56,804 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732422776804
2024-11-24T04:30:56,804 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 1 msec
2024-11-24T04:30:56,804 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,35553,1732422655941-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-24T04:30:56,804 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,35553,1732422655941-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T04:30:56,804 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,35553,1732422655941-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T04:30:56,805 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-fadb2fe0ba32:35553, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T04:30:56,805 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-24T04:30:56,805 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-24T04:30:56,806 DEBUG [master/fadb2fe0ba32:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-24T04:30:56,808 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.783sec
2024-11-24T04:30:56,808 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-24T04:30:56,808 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-24T04:30:56,808 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-24T04:30:56,808 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-24T04:30:56,808 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-24T04:30:56,808 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,35553,1732422655941-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-24T04:30:56,808 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,35553,1732422655941-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-24T04:30:56,811 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-24T04:30:56,811 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-24T04:30:56,811 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,35553,1732422655941-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T04:30:56,811 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:56,818 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:56,909 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e2a30ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-24T04:30:56,909 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fadb2fe0ba32,35553,-1 for getting cluster id
2024-11-24T04:30:56,909 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-24T04:30:56,911 DEBUG [HMaster-EventLoopGroup-12-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5685d04a-4df7-45da-9ee3-6eb5b42fb61e'
2024-11-24T04:30:56,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-24T04:30:56,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5685d04a-4df7-45da-9ee3-6eb5b42fb61e"

2024-11-24T04:30:56,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5178d4cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-24T04:30:56,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fadb2fe0ba32,35553,-1]
2024-11-24T04:30:56,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-24T04:30:56,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T04:30:56,913 INFO  [HMaster-EventLoopGroup-12-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43820, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-24T04:30:56,914 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30873421, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-24T04:30:56,915 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-24T04:30:56,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fadb2fe0ba32,41373,1732422655987, seqNum=-1]
2024-11-24T04:30:56,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-24T04:30:56,917 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33728, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-24T04:30:56,919 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=fadb2fe0ba32,35553,1732422655941
2024-11-24T04:30:56,919 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T04:30:56,921 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-24T04:30:56,922 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-24T04:30:56,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncConnectionImpl(321): The fetched master address is fadb2fe0ba32,35553,1732422655941
2024-11-24T04:30:56,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1924e272
2024-11-24T04:30:56,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-24T04:30:56,924 INFO  [HMaster-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43832, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-24T04:30:56,924 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35553 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-24T04:30:56,924 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35553 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-24T04:30:56,924 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35553 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testCompactionRecordDoesntBlockRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-24T04:30:56,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35553 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-24T04:30:56,927 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-11-24T04:30:56,927 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:30:56,927 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35553 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testCompactionRecordDoesntBlockRolling"
 procId is: 4
2024-11-24T04:30:56,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35553 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-24T04:30:56,929 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-24T04:30:56,935 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36061 is added to blk_1073741835_1011 (size=405)
2024-11-24T04:30:56,936 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45239 is added to blk_1073741835_1011 (size=405)
2024-11-24T04:30:56,937 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e42883f77bd200d47e2d35d98fdd65e0, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732422656924.e42883f77bd200d47e2d35d98fdd65e0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testCompactionRecordDoesntBlockRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28
2024-11-24T04:30:56,943 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45239 is added to blk_1073741836_1012 (size=88)
2024-11-24T04:30:56,943 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36061 is added to blk_1073741836_1012 (size=88)
2024-11-24T04:30:56,944 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732422656924.e42883f77bd200d47e2d35d98fdd65e0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T04:30:56,944 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1722): Closing e42883f77bd200d47e2d35d98fdd65e0, disabling compactions & flushes
2024-11-24T04:30:56,944 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732422656924.e42883f77bd200d47e2d35d98fdd65e0.
2024-11-24T04:30:56,944 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732422656924.e42883f77bd200d47e2d35d98fdd65e0.
2024-11-24T04:30:56,944 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732422656924.e42883f77bd200d47e2d35d98fdd65e0. after waiting 0 ms
2024-11-24T04:30:56,944 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732422656924.e42883f77bd200d47e2d35d98fdd65e0.
2024-11-24T04:30:56,944 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732422656924.e42883f77bd200d47e2d35d98fdd65e0.
2024-11-24T04:30:56,944 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for e42883f77bd200d47e2d35d98fdd65e0:
Waiting for close lock at 1732422656944Disabling compacts and flushes for region at 1732422656944Disabling writes for close at 1732422656944Writing region close event to WAL at 1732422656944Closed at 1732422656944
2024-11-24T04:30:56,946 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ADD_TO_META
2024-11-24T04:30:56,946 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732422656924.e42883f77bd200d47e2d35d98fdd65e0.","families":{"info":[{"qualifier":"regioninfo","vlen":87,"tag":[],"timestamp":"1732422656946"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732422656946"}]},"ts":"1732422656946"}
2024-11-24T04:30:56,949 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-24T04:30:56,950 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-24T04:30:56,950 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732422656950"}]},"ts":"1732422656950"}
2024-11-24T04:30:56,952 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLING in hbase:meta
2024-11-24T04:30:56,953 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=e42883f77bd200d47e2d35d98fdd65e0, ASSIGN}]
2024-11-24T04:30:56,954 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=e42883f77bd200d47e2d35d98fdd65e0, ASSIGN
2024-11-24T04:30:56,955 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=e42883f77bd200d47e2d35d98fdd65e0, ASSIGN; state=OFFLINE, location=fadb2fe0ba32,41373,1732422655987; forceNewPlan=false, retain=false
2024-11-24T04:30:57,106 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=e42883f77bd200d47e2d35d98fdd65e0, regionState=OPENING, regionLocation=fadb2fe0ba32,41373,1732422655987
2024-11-24T04:30:57,108 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-13-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=e42883f77bd200d47e2d35d98fdd65e0, ASSIGN because future has completed
2024-11-24T04:30:57,109 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure e42883f77bd200d47e2d35d98fdd65e0, server=fadb2fe0ba32,41373,1732422655987}]
2024-11-24T04:30:57,265 INFO  [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732422656924.e42883f77bd200d47e2d35d98fdd65e0.
2024-11-24T04:30:57,265 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => e42883f77bd200d47e2d35d98fdd65e0, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732422656924.e42883f77bd200d47e2d35d98fdd65e0.', STARTKEY => '', ENDKEY => ''}
2024-11-24T04:30:57,265 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testCompactionRecordDoesntBlockRolling e42883f77bd200d47e2d35d98fdd65e0
2024-11-24T04:30:57,265 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732422656924.e42883f77bd200d47e2d35d98fdd65e0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T04:30:57,265 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for e42883f77bd200d47e2d35d98fdd65e0
2024-11-24T04:30:57,265 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for e42883f77bd200d47e2d35d98fdd65e0
2024-11-24T04:30:57,266 INFO  [StoreOpener-e42883f77bd200d47e2d35d98fdd65e0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region e42883f77bd200d47e2d35d98fdd65e0 
2024-11-24T04:30:57,268 INFO  [StoreOpener-e42883f77bd200d47e2d35d98fdd65e0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e42883f77bd200d47e2d35d98fdd65e0 columnFamilyName info
2024-11-24T04:30:57,268 DEBUG [StoreOpener-e42883f77bd200d47e2d35d98fdd65e0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:30:57,268 INFO  [StoreOpener-e42883f77bd200d47e2d35d98fdd65e0-1 {}] regionserver.HStore(327): Store=e42883f77bd200d47e2d35d98fdd65e0/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-24T04:30:57,269 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for e42883f77bd200d47e2d35d98fdd65e0
2024-11-24T04:30:57,269 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e42883f77bd200d47e2d35d98fdd65e0
2024-11-24T04:30:57,269 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e42883f77bd200d47e2d35d98fdd65e0
2024-11-24T04:30:57,270 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for e42883f77bd200d47e2d35d98fdd65e0
2024-11-24T04:30:57,270 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for e42883f77bd200d47e2d35d98fdd65e0
2024-11-24T04:30:57,271 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for e42883f77bd200d47e2d35d98fdd65e0
2024-11-24T04:30:57,273 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e42883f77bd200d47e2d35d98fdd65e0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-24T04:30:57,273 INFO  [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened e42883f77bd200d47e2d35d98fdd65e0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=833336, jitterRate=0.05964256823062897}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-24T04:30:57,273 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e42883f77bd200d47e2d35d98fdd65e0
2024-11-24T04:30:57,274 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for e42883f77bd200d47e2d35d98fdd65e0:
Running coprocessor pre-open hook at 1732422657265Writing region info on filesystem at 1732422657265Initializing all the Stores at 1732422657266 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422657266Cleaning up temporary data from old regions at 1732422657270 (+4 ms)Running coprocessor post-open hooks at 1732422657273 (+3 ms)Region opened successfully at 1732422657274 (+1 ms)
2024-11-24T04:30:57,275 INFO  [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732422656924.e42883f77bd200d47e2d35d98fdd65e0., pid=6, masterSystemTime=1732422657261
2024-11-24T04:30:57,277 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732422656924.e42883f77bd200d47e2d35d98fdd65e0.
2024-11-24T04:30:57,277 INFO  [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732422656924.e42883f77bd200d47e2d35d98fdd65e0.
2024-11-24T04:30:57,278 INFO  [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=e42883f77bd200d47e2d35d98fdd65e0, regionState=OPEN, openSeqNum=2, regionLocation=fadb2fe0ba32,41373,1732422655987
2024-11-24T04:30:57,280 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-13-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure e42883f77bd200d47e2d35d98fdd65e0, server=fadb2fe0ba32,41373,1732422655987 because future has completed
2024-11-24T04:30:57,285 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-24T04:30:57,285 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure e42883f77bd200d47e2d35d98fdd65e0, server=fadb2fe0ba32,41373,1732422655987 in 173 msec
2024-11-24T04:30:57,287 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-24T04:30:57,287 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=e42883f77bd200d47e2d35d98fdd65e0, ASSIGN in 332 msec
2024-11-24T04:30:57,288 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-24T04:30:57,289 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732422657288"}]},"ts":"1732422657288"}
2024-11-24T04:30:57,291 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLED in hbase:meta
2024-11-24T04:30:57,292 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_POST_OPERATION
2024-11-24T04:30:57,293 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 367 msec
2024-11-24T04:30:57,811 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:57,819 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:58,812 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:58,819 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:59,812 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:30:59,819 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:00,813 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:00,820 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:01,768 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-24T04:31:01,769 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:31:01,770 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:31:01,770 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:31:01,770 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:31:01,770 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:31:01,770 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:31:01,784 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:31:01,784 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:31:01,784 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:31:01,784 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:31:01,785 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:31:01,785 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:31:01,788 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:31:01,788 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:31:01,788 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:31:01,791 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:31:01,813 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:01,820 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:02,236 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-24T04:31:02,237 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testCompactionRecordDoesntBlockRolling'
2024-11-24T04:31:02,814 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:02,821 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:03,815 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:03,821 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:04,815 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:04,822 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:05,816 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:05,822 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:06,266 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-24T04:31:06,266 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-11-24T04:31:06,267 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-24T04:31:06,267 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-11-24T04:31:06,267 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-24T04:31:06,267 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling Metrics about Tables on a single HBase RegionServer
2024-11-24T04:31:06,816 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:06,823 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:06,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35553 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-24T04:31:06,933 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-24T04:31:06,933 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testCompactionRecordDoesntBlockRolling,, stopping at row=TestLogRolling-testCompactionRecordDoesntBlockRolling ,, for max=2147483647 with caching=100
2024-11-24T04:31:06,936 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-24T04:31:06,936 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732422656924.e42883f77bd200d47e2d35d98fdd65e0.
2024-11-24T04:31:06,939 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testCompactionRecordDoesntBlockRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732422656924.e42883f77bd200d47e2d35d98fdd65e0., hostname=fadb2fe0ba32,41373,1732422655987, seqNum=2]
2024-11-24T04:31:06,946 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35553 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-24T04:31:06,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35553 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-24T04:31:06,952 INFO  [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-11-24T04:31:06,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35553 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7
2024-11-24T04:31:06,953 INFO  [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-11-24T04:31:06,954 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-11-24T04:31:07,114 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41373 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8
2024-11-24T04:31:07,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fadb2fe0ba32:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732422656924.e42883f77bd200d47e2d35d98fdd65e0.
2024-11-24T04:31:07,115 INFO  [RS_FLUSH_OPERATIONS-regionserver/fadb2fe0ba32:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing e42883f77bd200d47e2d35d98fdd65e0 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-24T04:31:07,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fadb2fe0ba32:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e42883f77bd200d47e2d35d98fdd65e0/.tmp/info/1e72f899eeab4d3e9523463f3ce2e38f is 1080, key is row0001/info:/1732422666940/Put/seqid=0
2024-11-24T04:31:07,136 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36061 is added to blk_1073741837_1013 (size=6033)
2024-11-24T04:31:07,136 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45239 is added to blk_1073741837_1013 (size=6033)
2024-11-24T04:31:07,137 INFO  [RS_FLUSH_OPERATIONS-regionserver/fadb2fe0ba32:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e42883f77bd200d47e2d35d98fdd65e0/.tmp/info/1e72f899eeab4d3e9523463f3ce2e38f
2024-11-24T04:31:07,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fadb2fe0ba32:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e42883f77bd200d47e2d35d98fdd65e0/.tmp/info/1e72f899eeab4d3e9523463f3ce2e38f as hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e42883f77bd200d47e2d35d98fdd65e0/info/1e72f899eeab4d3e9523463f3ce2e38f
2024-11-24T04:31:07,149 INFO  [RS_FLUSH_OPERATIONS-regionserver/fadb2fe0ba32:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e42883f77bd200d47e2d35d98fdd65e0/info/1e72f899eeab4d3e9523463f3ce2e38f, entries=1, sequenceid=5, filesize=5.9 K
2024-11-24T04:31:07,150 INFO  [RS_FLUSH_OPERATIONS-regionserver/fadb2fe0ba32:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for e42883f77bd200d47e2d35d98fdd65e0 in 35ms, sequenceid=5, compaction requested=false
2024-11-24T04:31:07,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fadb2fe0ba32:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for e42883f77bd200d47e2d35d98fdd65e0:

2024-11-24T04:31:07,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fadb2fe0ba32:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732422656924.e42883f77bd200d47e2d35d98fdd65e0.
2024-11-24T04:31:07,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fadb2fe0ba32:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8
2024-11-24T04:31:07,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35553 {}] master.HMaster(4169): Remote procedure done, pid=8
2024-11-24T04:31:07,157 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7
2024-11-24T04:31:07,157 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 201 msec
2024-11-24T04:31:07,160 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 211 msec
2024-11-24T04:31:07,817 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:07,823 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:08,817 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:08,823 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:09,818 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:09,824 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:10,818 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:10,824 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:11,819 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:11,825 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:12,820 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:12,825 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:13,820 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:13,826 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:14,821 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:14,826 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:15,821 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:15,826 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:16,822 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:16,827 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:16,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35553 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7
2024-11-24T04:31:16,973 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-24T04:31:16,976 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35553 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-24T04:31:16,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35553 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-24T04:31:16,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35553 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9
2024-11-24T04:31:16,978 INFO  [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-11-24T04:31:16,979 INFO  [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-11-24T04:31:16,979 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-11-24T04:31:17,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41373 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10
2024-11-24T04:31:17,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fadb2fe0ba32:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732422656924.e42883f77bd200d47e2d35d98fdd65e0.
2024-11-24T04:31:17,133 INFO  [RS_FLUSH_OPERATIONS-regionserver/fadb2fe0ba32:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing e42883f77bd200d47e2d35d98fdd65e0 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-24T04:31:17,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fadb2fe0ba32:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e42883f77bd200d47e2d35d98fdd65e0/.tmp/info/ea402ec4721c41328dd51daabe6fda26 is 1080, key is row0002/info:/1732422676974/Put/seqid=0
2024-11-24T04:31:17,142 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36061 is added to blk_1073741838_1014 (size=6033)
2024-11-24T04:31:17,142 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45239 is added to blk_1073741838_1014 (size=6033)
2024-11-24T04:31:17,142 INFO  [RS_FLUSH_OPERATIONS-regionserver/fadb2fe0ba32:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e42883f77bd200d47e2d35d98fdd65e0/.tmp/info/ea402ec4721c41328dd51daabe6fda26
2024-11-24T04:31:17,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fadb2fe0ba32:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e42883f77bd200d47e2d35d98fdd65e0/.tmp/info/ea402ec4721c41328dd51daabe6fda26 as hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e42883f77bd200d47e2d35d98fdd65e0/info/ea402ec4721c41328dd51daabe6fda26
2024-11-24T04:31:17,153 INFO  [RS_FLUSH_OPERATIONS-regionserver/fadb2fe0ba32:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e42883f77bd200d47e2d35d98fdd65e0/info/ea402ec4721c41328dd51daabe6fda26, entries=1, sequenceid=9, filesize=5.9 K
2024-11-24T04:31:17,154 INFO  [RS_FLUSH_OPERATIONS-regionserver/fadb2fe0ba32:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for e42883f77bd200d47e2d35d98fdd65e0 in 20ms, sequenceid=9, compaction requested=false
2024-11-24T04:31:17,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fadb2fe0ba32:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for e42883f77bd200d47e2d35d98fdd65e0:

2024-11-24T04:31:17,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fadb2fe0ba32:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732422656924.e42883f77bd200d47e2d35d98fdd65e0.
2024-11-24T04:31:17,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fadb2fe0ba32:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10
2024-11-24T04:31:17,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35553 {}] master.HMaster(4169): Remote procedure done, pid=10
2024-11-24T04:31:17,158 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9
2024-11-24T04:31:17,158 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 176 msec
2024-11-24T04:31:17,160 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 183 msec
2024-11-24T04:31:17,823 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:17,827 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:18,823 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:18,828 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:19,824 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:19,828 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:20,824 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:20,829 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:21,825 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:21,829 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:22,825 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:22,826 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749 after 68077ms
java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor202.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T04:31:22,830 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:22,830 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta after 68034ms
java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor202.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-24T04:31:23,826 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:23,830 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:24,827 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:24,831 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:25,827 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:25,831 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:25,925 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-24T04:31:26,828 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:26,831 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:27,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35553 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9
2024-11-24T04:31:27,073 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-24T04:31:27,076 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor fadb2fe0ba32%2C41373%2C1732422655987.1732422687076
2024-11-24T04:31:27,081 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:31:27,081 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:31:27,081 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:31:27,081 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:31:27,081 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:31:27,081 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/WALs/fadb2fe0ba32,41373,1732422655987/fadb2fe0ba32%2C41373%2C1732422655987.1732422656377 with entries=8, filesize=5.41 KB; new WAL /user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/WALs/fadb2fe0ba32,41373,1732422655987/fadb2fe0ba32%2C41373%2C1732422655987.1732422687076
2024-11-24T04:31:27,083 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42033:42033),(127.0.0.1/127.0.0.1:42935:42935)]
2024-11-24T04:31:27,083 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/WALs/fadb2fe0ba32,41373,1732422655987/fadb2fe0ba32%2C41373%2C1732422655987.1732422656377 is not closed yet, will try archiving it next time
2024-11-24T04:31:27,083 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36061 is added to blk_1073741833_1009 (size=5546)
2024-11-24T04:31:27,084 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45239 is added to blk_1073741833_1009 (size=5546)
2024-11-24T04:31:27,084 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35553 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-24T04:31:27,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35553 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-24T04:31:27,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35553 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11
2024-11-24T04:31:27,086 INFO  [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-11-24T04:31:27,087 INFO  [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-11-24T04:31:27,087 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-11-24T04:31:27,240 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41373 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12
2024-11-24T04:31:27,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fadb2fe0ba32:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732422656924.e42883f77bd200d47e2d35d98fdd65e0.
2024-11-24T04:31:27,241 INFO  [RS_FLUSH_OPERATIONS-regionserver/fadb2fe0ba32:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing e42883f77bd200d47e2d35d98fdd65e0 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-24T04:31:27,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fadb2fe0ba32:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e42883f77bd200d47e2d35d98fdd65e0/.tmp/info/d8d0be88db8f4899abd25a95a493233b is 1080, key is row0003/info:/1732422687074/Put/seqid=0
2024-11-24T04:31:27,249 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45239 is added to blk_1073741840_1016 (size=6033)
2024-11-24T04:31:27,250 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36061 is added to blk_1073741840_1016 (size=6033)
2024-11-24T04:31:27,250 INFO  [RS_FLUSH_OPERATIONS-regionserver/fadb2fe0ba32:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e42883f77bd200d47e2d35d98fdd65e0/.tmp/info/d8d0be88db8f4899abd25a95a493233b
2024-11-24T04:31:27,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fadb2fe0ba32:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e42883f77bd200d47e2d35d98fdd65e0/.tmp/info/d8d0be88db8f4899abd25a95a493233b as hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e42883f77bd200d47e2d35d98fdd65e0/info/d8d0be88db8f4899abd25a95a493233b
2024-11-24T04:31:27,260 INFO  [RS_FLUSH_OPERATIONS-regionserver/fadb2fe0ba32:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e42883f77bd200d47e2d35d98fdd65e0/info/d8d0be88db8f4899abd25a95a493233b, entries=1, sequenceid=13, filesize=5.9 K
2024-11-24T04:31:27,261 INFO  [RS_FLUSH_OPERATIONS-regionserver/fadb2fe0ba32:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for e42883f77bd200d47e2d35d98fdd65e0 in 20ms, sequenceid=13, compaction requested=true
2024-11-24T04:31:27,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fadb2fe0ba32:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for e42883f77bd200d47e2d35d98fdd65e0:

2024-11-24T04:31:27,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fadb2fe0ba32:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732422656924.e42883f77bd200d47e2d35d98fdd65e0.
2024-11-24T04:31:27,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fadb2fe0ba32:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12
2024-11-24T04:31:27,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35553 {}] master.HMaster(4169): Remote procedure done, pid=12
2024-11-24T04:31:27,265 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11
2024-11-24T04:31:27,265 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 176 msec
2024-11-24T04:31:27,268 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 182 msec
2024-11-24T04:31:27,828 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:27,832 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:28,829 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:28,832 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:29,830 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:29,833 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:30,830 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:30,833 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:31,831 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:31,834 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:32,831 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:32,834 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:33,832 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:33,835 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:34,832 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:34,835 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:35,833 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:35,835 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:36,811 INFO  [master/fadb2fe0ba32:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-11-24T04:31:36,811 INFO  [master/fadb2fe0ba32:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-11-24T04:31:36,834 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:36,836 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:37,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35553 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11
2024-11-24T04:31:37,183 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-24T04:31:37,183 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-24T04:31:37,185 DEBUG [Time-limited test {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 18099 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-24T04:31:37,185 DEBUG [Time-limited test {}] regionserver.HStore(1541): e42883f77bd200d47e2d35d98fdd65e0/info is initiating minor compaction (all files)
2024-11-24T04:31:37,185 INFO  [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-24T04:31:37,185 INFO  [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-24T04:31:37,185 INFO  [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of e42883f77bd200d47e2d35d98fdd65e0/info in TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732422656924.e42883f77bd200d47e2d35d98fdd65e0.
2024-11-24T04:31:37,185 INFO  [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e42883f77bd200d47e2d35d98fdd65e0/info/1e72f899eeab4d3e9523463f3ce2e38f, hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e42883f77bd200d47e2d35d98fdd65e0/info/ea402ec4721c41328dd51daabe6fda26, hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e42883f77bd200d47e2d35d98fdd65e0/info/d8d0be88db8f4899abd25a95a493233b] into tmpdir=hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e42883f77bd200d47e2d35d98fdd65e0/.tmp, totalSize=17.7 K
2024-11-24T04:31:37,186 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 1e72f899eeab4d3e9523463f3ce2e38f, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1732422666940
2024-11-24T04:31:37,186 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting ea402ec4721c41328dd51daabe6fda26, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=9, earliestPutTs=1732422676974
2024-11-24T04:31:37,186 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting d8d0be88db8f4899abd25a95a493233b, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732422687074
2024-11-24T04:31:37,198 INFO  [Time-limited test {}] throttle.PressureAwareThroughputController(145): e42883f77bd200d47e2d35d98fdd65e0#info#compaction#45 average throughput is 3.08 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-24T04:31:37,198 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e42883f77bd200d47e2d35d98fdd65e0/.tmp/info/9967c9938fe54d48a53fe95ad7dc76f7 is 1080, key is row0001/info:/1732422666940/Put/seqid=0
2024-11-24T04:31:37,204 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36061 is added to blk_1073741841_1017 (size=8296)
2024-11-24T04:31:37,204 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45239 is added to blk_1073741841_1017 (size=8296)
2024-11-24T04:31:37,210 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e42883f77bd200d47e2d35d98fdd65e0/.tmp/info/9967c9938fe54d48a53fe95ad7dc76f7 as hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e42883f77bd200d47e2d35d98fdd65e0/info/9967c9938fe54d48a53fe95ad7dc76f7
2024-11-24T04:31:37,216 INFO  [Time-limited test {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e42883f77bd200d47e2d35d98fdd65e0/info of e42883f77bd200d47e2d35d98fdd65e0 into 9967c9938fe54d48a53fe95ad7dc76f7(size=8.1 K), total size for store is 8.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-24T04:31:37,216 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for e42883f77bd200d47e2d35d98fdd65e0:

2024-11-24T04:31:37,219 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor fadb2fe0ba32%2C41373%2C1732422655987.1732422697219
2024-11-24T04:31:37,225 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:31:37,225 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:31:37,225 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:31:37,226 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:31:37,226 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:31:37,226 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/WALs/fadb2fe0ba32,41373,1732422655987/fadb2fe0ba32%2C41373%2C1732422655987.1732422687076 with entries=4, filesize=2.45 KB; new WAL /user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/WALs/fadb2fe0ba32,41373,1732422655987/fadb2fe0ba32%2C41373%2C1732422655987.1732422697219
2024-11-24T04:31:37,226 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42935:42935),(127.0.0.1/127.0.0.1:42033:42033)]
2024-11-24T04:31:37,226 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/WALs/fadb2fe0ba32,41373,1732422655987/fadb2fe0ba32%2C41373%2C1732422655987.1732422687076 is not closed yet, will try archiving it next time
2024-11-24T04:31:37,227 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/WALs/fadb2fe0ba32,41373,1732422655987/fadb2fe0ba32%2C41373%2C1732422655987.1732422656377 to hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/oldWALs/fadb2fe0ba32%2C41373%2C1732422655987.1732422656377
2024-11-24T04:31:37,227 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36061 is added to blk_1073741839_1015 (size=2520)
2024-11-24T04:31:37,227 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45239 is added to blk_1073741839_1015 (size=2520)
2024-11-24T04:31:37,228 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35553 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-24T04:31:37,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35553 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-24T04:31:37,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35553 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13
2024-11-24T04:31:37,230 INFO  [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-11-24T04:31:37,231 INFO  [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-11-24T04:31:37,232 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-11-24T04:31:37,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41373 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14
2024-11-24T04:31:37,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fadb2fe0ba32:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732422656924.e42883f77bd200d47e2d35d98fdd65e0.
2024-11-24T04:31:37,385 INFO  [RS_FLUSH_OPERATIONS-regionserver/fadb2fe0ba32:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing e42883f77bd200d47e2d35d98fdd65e0 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-24T04:31:37,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fadb2fe0ba32:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e42883f77bd200d47e2d35d98fdd65e0/.tmp/info/44ea3e84fced4586a79b77d0bbc6ad66 is 1080, key is row0000/info:/1732422697218/Put/seqid=0
2024-11-24T04:31:37,394 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36061 is added to blk_1073741843_1019 (size=6033)
2024-11-24T04:31:37,394 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45239 is added to blk_1073741843_1019 (size=6033)
2024-11-24T04:31:37,395 INFO  [RS_FLUSH_OPERATIONS-regionserver/fadb2fe0ba32:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e42883f77bd200d47e2d35d98fdd65e0/.tmp/info/44ea3e84fced4586a79b77d0bbc6ad66
2024-11-24T04:31:37,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fadb2fe0ba32:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e42883f77bd200d47e2d35d98fdd65e0/.tmp/info/44ea3e84fced4586a79b77d0bbc6ad66 as hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e42883f77bd200d47e2d35d98fdd65e0/info/44ea3e84fced4586a79b77d0bbc6ad66
2024-11-24T04:31:37,405 INFO  [RS_FLUSH_OPERATIONS-regionserver/fadb2fe0ba32:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e42883f77bd200d47e2d35d98fdd65e0/info/44ea3e84fced4586a79b77d0bbc6ad66, entries=1, sequenceid=18, filesize=5.9 K
2024-11-24T04:31:37,406 INFO  [RS_FLUSH_OPERATIONS-regionserver/fadb2fe0ba32:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for e42883f77bd200d47e2d35d98fdd65e0 in 21ms, sequenceid=18, compaction requested=false
2024-11-24T04:31:37,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fadb2fe0ba32:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for e42883f77bd200d47e2d35d98fdd65e0:

2024-11-24T04:31:37,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fadb2fe0ba32:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732422656924.e42883f77bd200d47e2d35d98fdd65e0.
2024-11-24T04:31:37,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fadb2fe0ba32:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14
2024-11-24T04:31:37,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35553 {}] master.HMaster(4169): Remote procedure done, pid=14
2024-11-24T04:31:37,410 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13
2024-11-24T04:31:37,410 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 176 msec
2024-11-24T04:31:37,413 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 183 msec
2024-11-24T04:31:37,834 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:37,836 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:38,835 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:38,837 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:39,835 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:39,837 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:40,836 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:40,837 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:41,836 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:41,838 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:42,265 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region e42883f77bd200d47e2d35d98fdd65e0, had cached 0 bytes from a total of 14329
2024-11-24T04:31:42,837 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:42,838 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:43,837 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:43,839 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:44,838 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:44,839 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:45,838 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:45,840 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:46,839 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:46,840 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:47,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35553 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13
2024-11-24T04:31:47,323 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-24T04:31:47,325 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor fadb2fe0ba32%2C41373%2C1732422655987.1732422707325
2024-11-24T04:31:47,331 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:31:47,331 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:31:47,331 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:31:47,331 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:31:47,331 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:31:47,332 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/WALs/fadb2fe0ba32,41373,1732422655987/fadb2fe0ba32%2C41373%2C1732422655987.1732422697219 with entries=3, filesize=1.97 KB; new WAL /user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/WALs/fadb2fe0ba32,41373,1732422655987/fadb2fe0ba32%2C41373%2C1732422655987.1732422707325
2024-11-24T04:31:47,332 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42935:42935),(127.0.0.1/127.0.0.1:42033:42033)]
2024-11-24T04:31:47,332 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/WALs/fadb2fe0ba32,41373,1732422655987/fadb2fe0ba32%2C41373%2C1732422655987.1732422697219 is not closed yet, will try archiving it next time
2024-11-24T04:31:47,332 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-24T04:31:47,332 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/WALs/fadb2fe0ba32,41373,1732422655987/fadb2fe0ba32%2C41373%2C1732422655987.1732422687076 to hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/oldWALs/fadb2fe0ba32%2C41373%2C1732422655987.1732422687076
2024-11-24T04:31:47,333 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-24T04:31:47,333 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-24T04:31:47,333 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T04:31:47,333 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T04:31:47,333 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-24T04:31:47,333 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1963781635, stopped=false
2024-11-24T04:31:47,333 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=fadb2fe0ba32,35553,1732422655941
2024-11-24T04:31:47,333 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45239 is added to blk_1073741842_1018 (size=2026)
2024-11-24T04:31:47,333 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-24T04:31:47,334 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36061 is added to blk_1073741842_1018 (size=2026)
2024-11-24T04:31:47,335 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35553-0x101483dca470000, quorum=127.0.0.1:49920, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-24T04:31:47,335 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41373-0x101483dca470001, quorum=127.0.0.1:49920, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-24T04:31:47,335 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35553-0x101483dca470000, quorum=127.0.0.1:49920, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:31:47,335 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41373-0x101483dca470001, quorum=127.0.0.1:49920, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:31:47,335 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-24T04:31:47,335 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-24T04:31:47,335 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-24T04:31:47,335 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T04:31:47,336 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'fadb2fe0ba32,41373,1732422655987' *****
2024-11-24T04:31:47,336 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-24T04:31:47,336 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35553-0x101483dca470000, quorum=127.0.0.1:49920, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-24T04:31:47,336 INFO  [RS:0;fadb2fe0ba32:41373 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-24T04:31:47,336 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-24T04:31:47,336 INFO  [RS:0;fadb2fe0ba32:41373 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-24T04:31:47,336 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41373-0x101483dca470001, quorum=127.0.0.1:49920, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-24T04:31:47,336 INFO  [RS:0;fadb2fe0ba32:41373 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-24T04:31:47,336 INFO  [RS:0;fadb2fe0ba32:41373 {}] regionserver.HRegionServer(3091): Received CLOSE for e42883f77bd200d47e2d35d98fdd65e0
2024-11-24T04:31:47,337 INFO  [RS:0;fadb2fe0ba32:41373 {}] regionserver.HRegionServer(959): stopping server fadb2fe0ba32,41373,1732422655987
2024-11-24T04:31:47,337 INFO  [RS:0;fadb2fe0ba32:41373 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-24T04:31:47,337 INFO  [RS:0;fadb2fe0ba32:41373 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;fadb2fe0ba32:41373.
2024-11-24T04:31:47,337 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e42883f77bd200d47e2d35d98fdd65e0, disabling compactions & flushes
2024-11-24T04:31:47,337 DEBUG [RS:0;fadb2fe0ba32:41373 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-24T04:31:47,337 INFO  [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732422656924.e42883f77bd200d47e2d35d98fdd65e0.
2024-11-24T04:31:47,337 DEBUG [RS:0;fadb2fe0ba32:41373 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T04:31:47,337 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732422656924.e42883f77bd200d47e2d35d98fdd65e0.
2024-11-24T04:31:47,337 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732422656924.e42883f77bd200d47e2d35d98fdd65e0. after waiting 0 ms
2024-11-24T04:31:47,337 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732422656924.e42883f77bd200d47e2d35d98fdd65e0.
2024-11-24T04:31:47,337 INFO  [RS:0;fadb2fe0ba32:41373 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-24T04:31:47,337 INFO  [RS:0;fadb2fe0ba32:41373 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-24T04:31:47,337 INFO  [RS:0;fadb2fe0ba32:41373 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-24T04:31:47,337 INFO  [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing e42883f77bd200d47e2d35d98fdd65e0 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-24T04:31:47,337 INFO  [RS:0;fadb2fe0ba32:41373 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-24T04:31:47,337 INFO  [RS:0;fadb2fe0ba32:41373 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-11-24T04:31:47,337 DEBUG [RS:0;fadb2fe0ba32:41373 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, e42883f77bd200d47e2d35d98fdd65e0=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732422656924.e42883f77bd200d47e2d35d98fdd65e0.}
2024-11-24T04:31:47,337 DEBUG [RS:0;fadb2fe0ba32:41373 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, e42883f77bd200d47e2d35d98fdd65e0
2024-11-24T04:31:47,337 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-24T04:31:47,337 INFO  [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-24T04:31:47,338 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-24T04:31:47,338 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-24T04:31:47,338 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-24T04:31:47,338 INFO  [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.89 KB heapSize=3.91 KB
2024-11-24T04:31:47,342 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e42883f77bd200d47e2d35d98fdd65e0/.tmp/info/19d5abf2542844aeaaa7892146b93df9 is 1080, key is row0001/info:/1732422707324/Put/seqid=0
2024-11-24T04:31:47,350 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45239 is added to blk_1073741845_1021 (size=6033)
2024-11-24T04:31:47,350 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36061 is added to blk_1073741845_1021 (size=6033)
2024-11-24T04:31:47,350 INFO  [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=22 (bloomFilter=true), to=hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e42883f77bd200d47e2d35d98fdd65e0/.tmp/info/19d5abf2542844aeaaa7892146b93df9
2024-11-24T04:31:47,356 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/data/hbase/meta/1588230740/.tmp/info/4dc7282577a64363b3eb1317165090ad is 227, key is TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732422656924.e42883f77bd200d47e2d35d98fdd65e0./info:regioninfo/1732422657278/Put/seqid=0
2024-11-24T04:31:47,356 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e42883f77bd200d47e2d35d98fdd65e0/.tmp/info/19d5abf2542844aeaaa7892146b93df9 as hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e42883f77bd200d47e2d35d98fdd65e0/info/19d5abf2542844aeaaa7892146b93df9
2024-11-24T04:31:47,360 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45239 is added to blk_1073741846_1022 (size=7308)
2024-11-24T04:31:47,360 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36061 is added to blk_1073741846_1022 (size=7308)
2024-11-24T04:31:47,361 INFO  [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/data/hbase/meta/1588230740/.tmp/info/4dc7282577a64363b3eb1317165090ad
2024-11-24T04:31:47,362 INFO  [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e42883f77bd200d47e2d35d98fdd65e0/info/19d5abf2542844aeaaa7892146b93df9, entries=1, sequenceid=22, filesize=5.9 K
2024-11-24T04:31:47,363 INFO  [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for e42883f77bd200d47e2d35d98fdd65e0 in 26ms, sequenceid=22, compaction requested=true
2024-11-24T04:31:47,364 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732422656924.e42883f77bd200d47e2d35d98fdd65e0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e42883f77bd200d47e2d35d98fdd65e0/info/1e72f899eeab4d3e9523463f3ce2e38f, hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e42883f77bd200d47e2d35d98fdd65e0/info/ea402ec4721c41328dd51daabe6fda26, hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e42883f77bd200d47e2d35d98fdd65e0/info/d8d0be88db8f4899abd25a95a493233b] to archive
2024-11-24T04:31:47,364 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732422656924.e42883f77bd200d47e2d35d98fdd65e0.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-24T04:31:47,366 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732422656924.e42883f77bd200d47e2d35d98fdd65e0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e42883f77bd200d47e2d35d98fdd65e0/info/1e72f899eeab4d3e9523463f3ce2e38f to hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e42883f77bd200d47e2d35d98fdd65e0/info/1e72f899eeab4d3e9523463f3ce2e38f
2024-11-24T04:31:47,367 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732422656924.e42883f77bd200d47e2d35d98fdd65e0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e42883f77bd200d47e2d35d98fdd65e0/info/ea402ec4721c41328dd51daabe6fda26 to hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e42883f77bd200d47e2d35d98fdd65e0/info/ea402ec4721c41328dd51daabe6fda26
2024-11-24T04:31:47,368 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732422656924.e42883f77bd200d47e2d35d98fdd65e0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e42883f77bd200d47e2d35d98fdd65e0/info/d8d0be88db8f4899abd25a95a493233b to hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e42883f77bd200d47e2d35d98fdd65e0/info/d8d0be88db8f4899abd25a95a493233b
2024-11-24T04:31:47,369 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732422656924.e42883f77bd200d47e2d35d98fdd65e0.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=fadb2fe0ba32:35553 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 16 more
2024-11-24T04:31:47,369 WARN  [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732422656924.e42883f77bd200d47e2d35d98fdd65e0.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [1e72f899eeab4d3e9523463f3ce2e38f=6033, ea402ec4721c41328dd51daabe6fda26=6033, d8d0be88db8f4899abd25a95a493233b=6033]
2024-11-24T04:31:47,372 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e42883f77bd200d47e2d35d98fdd65e0/recovered.edits/25.seqid, newMaxSeqId=25, maxSeqId=1
2024-11-24T04:31:47,373 INFO  [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732422656924.e42883f77bd200d47e2d35d98fdd65e0.
2024-11-24T04:31:47,373 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e42883f77bd200d47e2d35d98fdd65e0:
Waiting for close lock at 1732422707337Running coprocessor pre-close hooks at 1732422707337Disabling compacts and flushes for region at 1732422707337Disabling writes for close at 1732422707337Obtaining lock to block concurrent updates at 1732422707337Preparing flush snapshotting stores in e42883f77bd200d47e2d35d98fdd65e0 at 1732422707337Finished memstore snapshotting TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732422656924.e42883f77bd200d47e2d35d98fdd65e0., syncing WAL and waiting on mvcc, flushsize=dataSize=1076, getHeapSize=1392, getOffHeapSize=0, getCellsCount=1 at 1732422707337Flushing stores of TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732422656924.e42883f77bd200d47e2d35d98fdd65e0. at 1732422707338 (+1 ms)Flushing e42883f77bd200d47e2d35d98fdd65e0/info: creating writer at 1732422707338Flushing e42883f77bd200d47e2d35d98fdd65e0/info: appending metadata at 1732422707342 (+4 ms)Flushing e42883f77bd200d47e2d35d98fdd65e0/info: closing flushed file at 1732422707342Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6d6ea6f0: reopening flushed file at 1732422707355 (+13 ms)Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for e42883f77bd200d47e2d35d98fdd65e0 in 26ms, sequenceid=22, compaction requested=true at 1732422707363 (+8 ms)Writing region close event to WAL at 1732422707369 (+6 ms)Running coprocessor post-close hooks at 1732422707373 (+4 ms)Closed at 1732422707373
2024-11-24T04:31:47,373 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732422656924.e42883f77bd200d47e2d35d98fdd65e0.
2024-11-24T04:31:47,381 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/data/hbase/meta/1588230740/.tmp/ns/c9f07fd8a4c14d0cb0251b8b495e1551 is 43, key is default/ns:d/1732422656797/Put/seqid=0
2024-11-24T04:31:47,385 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36061 is added to blk_1073741847_1023 (size=5153)
2024-11-24T04:31:47,385 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45239 is added to blk_1073741847_1023 (size=5153)
2024-11-24T04:31:47,385 INFO  [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/data/hbase/meta/1588230740/.tmp/ns/c9f07fd8a4c14d0cb0251b8b495e1551
2024-11-24T04:31:47,404 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/data/hbase/meta/1588230740/.tmp/table/ed02b2f467794a668beb0f6fd894bf55 is 89, key is TestLogRolling-testCompactionRecordDoesntBlockRolling/table:state/1732422657288/Put/seqid=0
2024-11-24T04:31:47,408 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36061 is added to blk_1073741848_1024 (size=5508)
2024-11-24T04:31:47,409 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45239 is added to blk_1073741848_1024 (size=5508)
2024-11-24T04:31:47,409 INFO  [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=170 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/data/hbase/meta/1588230740/.tmp/table/ed02b2f467794a668beb0f6fd894bf55
2024-11-24T04:31:47,414 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/data/hbase/meta/1588230740/.tmp/info/4dc7282577a64363b3eb1317165090ad as hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/data/hbase/meta/1588230740/info/4dc7282577a64363b3eb1317165090ad
2024-11-24T04:31:47,419 INFO  [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/data/hbase/meta/1588230740/info/4dc7282577a64363b3eb1317165090ad, entries=10, sequenceid=11, filesize=7.1 K
2024-11-24T04:31:47,419 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/data/hbase/meta/1588230740/.tmp/ns/c9f07fd8a4c14d0cb0251b8b495e1551 as hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/data/hbase/meta/1588230740/ns/c9f07fd8a4c14d0cb0251b8b495e1551
2024-11-24T04:31:47,424 INFO  [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/data/hbase/meta/1588230740/ns/c9f07fd8a4c14d0cb0251b8b495e1551, entries=2, sequenceid=11, filesize=5.0 K
2024-11-24T04:31:47,425 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/data/hbase/meta/1588230740/.tmp/table/ed02b2f467794a668beb0f6fd894bf55 as hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/data/hbase/meta/1588230740/table/ed02b2f467794a668beb0f6fd894bf55
2024-11-24T04:31:47,429 INFO  [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/data/hbase/meta/1588230740/table/ed02b2f467794a668beb0f6fd894bf55, entries=2, sequenceid=11, filesize=5.4 K
2024-11-24T04:31:47,430 INFO  [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.89 KB/1932, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 92ms, sequenceid=11, compaction requested=false
2024-11-24T04:31:47,434 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-11-24T04:31:47,435 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-24T04:31:47,435 INFO  [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-24T04:31:47,435 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732422707337Running coprocessor pre-close hooks at 1732422707337Disabling compacts and flushes for region at 1732422707337Disabling writes for close at 1732422707338 (+1 ms)Obtaining lock to block concurrent updates at 1732422707338Preparing flush snapshotting stores in 1588230740 at 1732422707338Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1932, getHeapSize=3936, getOffHeapSize=0, getCellsCount=14 at 1732422707338Flushing stores of hbase:meta,,1.1588230740 at 1732422707339 (+1 ms)Flushing 1588230740/info: creating writer at 1732422707339Flushing 1588230740/info: appending metadata at 1732422707355 (+16 ms)Flushing 1588230740/info: closing flushed file at 1732422707355Flushing 1588230740/ns: creating writer at 1732422707366 (+11 ms)Flushing 1588230740/ns: appending metadata at 1732422707380 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1732422707380Flushing 1588230740/table: creating writer at 1732422707390 (+10 ms)Flushing 1588230740/table: appending metadata at 1732422707404 (+14 ms)Flushing 1588230740/table: closing flushed file at 1732422707404Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@66573a65: reopening flushed file at 1732422707413 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@67a3e815: reopening flushed file at 1732422707419 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3b926837: reopening flushed file at 1732422707424 (+5 ms)Finished flush of dataSize ~1.89 KB/1932, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 92ms, sequenceid=11, compaction requested=false at 1732422707430 (+6 ms)Writing region close event to WAL at 1732422707431 (+1 ms)Running coprocessor post-close hooks at 1732422707435 (+4 ms)Closed at 1732422707435
2024-11-24T04:31:47,435 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-24T04:31:47,538 INFO  [RS:0;fadb2fe0ba32:41373 {}] regionserver.HRegionServer(976): stopping server fadb2fe0ba32,41373,1732422655987; all regions closed.
2024-11-24T04:31:47,538 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:31:47,538 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:31:47,538 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:31:47,538 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:31:47,538 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:31:47,540 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36061 is added to blk_1073741834_1010 (size=3306)
2024-11-24T04:31:47,541 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45239 is added to blk_1073741834_1010 (size=3306)
2024-11-24T04:31:47,543 DEBUG [RS:0;fadb2fe0ba32:41373 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/oldWALs
2024-11-24T04:31:47,543 INFO  [RS:0;fadb2fe0ba32:41373 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog fadb2fe0ba32%2C41373%2C1732422655987.meta:.meta(num 1732422656750)
2024-11-24T04:31:47,543 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:31:47,543 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:31:47,543 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:31:47,543 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:31:47,543 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:31:47,545 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45239 is added to blk_1073741844_1020 (size=1252)
2024-11-24T04:31:47,545 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36061 is added to blk_1073741844_1020 (size=1252)
2024-11-24T04:31:47,548 DEBUG [RS:0;fadb2fe0ba32:41373 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/oldWALs
2024-11-24T04:31:47,548 INFO  [RS:0;fadb2fe0ba32:41373 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog fadb2fe0ba32%2C41373%2C1732422655987:(num 1732422707325)
2024-11-24T04:31:47,548 DEBUG [RS:0;fadb2fe0ba32:41373 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T04:31:47,548 INFO  [RS:0;fadb2fe0ba32:41373 {}] regionserver.LeaseManager(133): Closed leases
2024-11-24T04:31:47,548 INFO  [RS:0;fadb2fe0ba32:41373 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-24T04:31:47,548 INFO  [RS:0;fadb2fe0ba32:41373 {}] hbase.ChoreService(370): Chore service for: regionserver/fadb2fe0ba32:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-24T04:31:47,548 INFO  [RS:0;fadb2fe0ba32:41373 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-24T04:31:47,548 INFO  [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-24T04:31:47,548 INFO  [RS:0;fadb2fe0ba32:41373 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41373
2024-11-24T04:31:47,552 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41373-0x101483dca470001, quorum=127.0.0.1:49920, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/fadb2fe0ba32,41373,1732422655987
2024-11-24T04:31:47,552 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35553-0x101483dca470000, quorum=127.0.0.1:49920, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-24T04:31:47,552 INFO  [RS:0;fadb2fe0ba32:41373 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-24T04:31:47,552 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [fadb2fe0ba32,41373,1732422655987]
2024-11-24T04:31:47,554 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/fadb2fe0ba32,41373,1732422655987 already deleted, retry=false
2024-11-24T04:31:47,554 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; fadb2fe0ba32,41373,1732422655987 expired; onlineServers=0
2024-11-24T04:31:47,554 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'fadb2fe0ba32,35553,1732422655941' *****
2024-11-24T04:31:47,554 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-24T04:31:47,554 INFO  [M:0;fadb2fe0ba32:35553 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-24T04:31:47,554 INFO  [M:0;fadb2fe0ba32:35553 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-24T04:31:47,554 DEBUG [M:0;fadb2fe0ba32:35553 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-24T04:31:47,554 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-24T04:31:47,554 DEBUG [M:0;fadb2fe0ba32:35553 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-24T04:31:47,554 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster-HFileCleaner.large.0-1732422656146 {}] cleaner.HFileCleaner(306): Exit Thread[master/fadb2fe0ba32:0:becomeActiveMaster-HFileCleaner.large.0-1732422656146,5,FailOnTimeoutGroup]
2024-11-24T04:31:47,554 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster-HFileCleaner.small.0-1732422656146 {}] cleaner.HFileCleaner(306): Exit Thread[master/fadb2fe0ba32:0:becomeActiveMaster-HFileCleaner.small.0-1732422656146,5,FailOnTimeoutGroup]
2024-11-24T04:31:47,554 INFO  [M:0;fadb2fe0ba32:35553 {}] hbase.ChoreService(370): Chore service for: master/fadb2fe0ba32:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-24T04:31:47,554 INFO  [M:0;fadb2fe0ba32:35553 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-24T04:31:47,555 DEBUG [M:0;fadb2fe0ba32:35553 {}] master.HMaster(1795): Stopping service threads
2024-11-24T04:31:47,555 INFO  [M:0;fadb2fe0ba32:35553 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-24T04:31:47,555 INFO  [M:0;fadb2fe0ba32:35553 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-24T04:31:47,555 INFO  [M:0;fadb2fe0ba32:35553 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-24T04:31:47,555 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-24T04:31:47,556 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35553-0x101483dca470000, quorum=127.0.0.1:49920, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-24T04:31:47,556 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35553-0x101483dca470000, quorum=127.0.0.1:49920, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:31:47,556 DEBUG [M:0;fadb2fe0ba32:35553 {}] zookeeper.ZKUtil(347): master:35553-0x101483dca470000, quorum=127.0.0.1:49920, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-24T04:31:47,556 WARN  [M:0;fadb2fe0ba32:35553 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-24T04:31:47,556 INFO  [M:0;fadb2fe0ba32:35553 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/.lastflushedseqids
2024-11-24T04:31:47,561 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45239 is added to blk_1073741849_1025 (size=130)
2024-11-24T04:31:47,561 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36061 is added to blk_1073741849_1025 (size=130)
2024-11-24T04:31:47,562 INFO  [M:0;fadb2fe0ba32:35553 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-24T04:31:47,562 INFO  [M:0;fadb2fe0ba32:35553 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-24T04:31:47,562 DEBUG [M:0;fadb2fe0ba32:35553 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-24T04:31:47,562 INFO  [M:0;fadb2fe0ba32:35553 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T04:31:47,562 DEBUG [M:0;fadb2fe0ba32:35553 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T04:31:47,562 DEBUG [M:0;fadb2fe0ba32:35553 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-24T04:31:47,562 DEBUG [M:0;fadb2fe0ba32:35553 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T04:31:47,562 INFO  [M:0;fadb2fe0ba32:35553 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=43.59 KB heapSize=55 KB
2024-11-24T04:31:47,578 DEBUG [M:0;fadb2fe0ba32:35553 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/28aa1d8a4f9a4d41b14af50663bb5703 is 82, key is hbase:meta,,1/info:regioninfo/1732422656781/Put/seqid=0
2024-11-24T04:31:47,581 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36061 is added to blk_1073741850_1026 (size=5672)
2024-11-24T04:31:47,582 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45239 is added to blk_1073741850_1026 (size=5672)
2024-11-24T04:31:47,582 INFO  [M:0;fadb2fe0ba32:35553 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/28aa1d8a4f9a4d41b14af50663bb5703
2024-11-24T04:31:47,600 DEBUG [M:0;fadb2fe0ba32:35553 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a279710bcd554600aa9f31b1f87e883a is 798, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732422657293/Put/seqid=0
2024-11-24T04:31:47,604 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45239 is added to blk_1073741851_1027 (size=7823)
2024-11-24T04:31:47,604 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36061 is added to blk_1073741851_1027 (size=7823)
2024-11-24T04:31:47,605 INFO  [M:0;fadb2fe0ba32:35553 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.99 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a279710bcd554600aa9f31b1f87e883a
2024-11-24T04:31:47,609 INFO  [M:0;fadb2fe0ba32:35553 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a279710bcd554600aa9f31b1f87e883a
2024-11-24T04:31:47,623 DEBUG [M:0;fadb2fe0ba32:35553 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/021c97311b9444dab83e8a3f66720d0a is 69, key is fadb2fe0ba32,41373,1732422655987/rs:state/1732422656229/Put/seqid=0
2024-11-24T04:31:47,627 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45239 is added to blk_1073741852_1028 (size=5156)
2024-11-24T04:31:47,627 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36061 is added to blk_1073741852_1028 (size=5156)
2024-11-24T04:31:47,627 INFO  [M:0;fadb2fe0ba32:35553 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/021c97311b9444dab83e8a3f66720d0a
2024-11-24T04:31:47,646 DEBUG [M:0;fadb2fe0ba32:35553 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/7fb9b64873bc44268f46e932218d04fe is 52, key is load_balancer_on/state:d/1732422656920/Put/seqid=0
2024-11-24T04:31:47,650 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36061 is added to blk_1073741853_1029 (size=5056)
2024-11-24T04:31:47,651 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45239 is added to blk_1073741853_1029 (size=5056)
2024-11-24T04:31:47,651 INFO  [M:0;fadb2fe0ba32:35553 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/7fb9b64873bc44268f46e932218d04fe
2024-11-24T04:31:47,653 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41373-0x101483dca470001, quorum=127.0.0.1:49920, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-24T04:31:47,653 INFO  [RS:0;fadb2fe0ba32:41373 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-24T04:31:47,653 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41373-0x101483dca470001, quorum=127.0.0.1:49920, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-24T04:31:47,653 INFO  [RS:0;fadb2fe0ba32:41373 {}] regionserver.HRegionServer(1031): Exiting; stopping=fadb2fe0ba32,41373,1732422655987; zookeeper connection closed.
2024-11-24T04:31:47,654 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4199e6b5 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4199e6b5
2024-11-24T04:31:47,654 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-24T04:31:47,656 DEBUG [M:0;fadb2fe0ba32:35553 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/28aa1d8a4f9a4d41b14af50663bb5703 as hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/28aa1d8a4f9a4d41b14af50663bb5703
2024-11-24T04:31:47,660 INFO  [M:0;fadb2fe0ba32:35553 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/28aa1d8a4f9a4d41b14af50663bb5703, entries=8, sequenceid=121, filesize=5.5 K
2024-11-24T04:31:47,661 DEBUG [M:0;fadb2fe0ba32:35553 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a279710bcd554600aa9f31b1f87e883a as hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a279710bcd554600aa9f31b1f87e883a
2024-11-24T04:31:47,664 INFO  [M:0;fadb2fe0ba32:35553 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a279710bcd554600aa9f31b1f87e883a
2024-11-24T04:31:47,665 INFO  [M:0;fadb2fe0ba32:35553 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a279710bcd554600aa9f31b1f87e883a, entries=14, sequenceid=121, filesize=7.6 K
2024-11-24T04:31:47,665 DEBUG [M:0;fadb2fe0ba32:35553 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/021c97311b9444dab83e8a3f66720d0a as hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/021c97311b9444dab83e8a3f66720d0a
2024-11-24T04:31:47,669 INFO  [M:0;fadb2fe0ba32:35553 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/021c97311b9444dab83e8a3f66720d0a, entries=1, sequenceid=121, filesize=5.0 K
2024-11-24T04:31:47,670 DEBUG [M:0;fadb2fe0ba32:35553 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/7fb9b64873bc44268f46e932218d04fe as hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/7fb9b64873bc44268f46e932218d04fe
2024-11-24T04:31:47,674 INFO  [M:0;fadb2fe0ba32:35553 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35273/user/jenkins/test-data/6b5f86c4-b1ea-7b9c-cd77-47a9227c8b28/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/7fb9b64873bc44268f46e932218d04fe, entries=1, sequenceid=121, filesize=4.9 K
2024-11-24T04:31:47,675 INFO  [M:0;fadb2fe0ba32:35553 {}] regionserver.HRegion(3140): Finished flush of dataSize ~43.59 KB/44638, heapSize ~54.94 KB/56256, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 113ms, sequenceid=121, compaction requested=false
2024-11-24T04:31:47,677 INFO  [M:0;fadb2fe0ba32:35553 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T04:31:47,677 DEBUG [M:0;fadb2fe0ba32:35553 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732422707562Disabling compacts and flushes for region at 1732422707562Disabling writes for close at 1732422707562Obtaining lock to block concurrent updates at 1732422707562Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732422707562Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=44638, getHeapSize=56256, getOffHeapSize=0, getCellsCount=140 at 1732422707562Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732422707563 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732422707563Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732422707577 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732422707577Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732422707586 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732422707600 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732422707600Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732422707609 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732422707622 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732422707622Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732422707632 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732422707646 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732422707646Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2dd0c9fe: reopening flushed file at 1732422707655 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@328e7485: reopening flushed file at 1732422707660 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@796126e4: reopening flushed file at 1732422707665 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@47a91e34: reopening flushed file at 1732422707669 (+4 ms)Finished flush of dataSize ~43.59 KB/44638, heapSize ~54.94 KB/56256, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 113ms, sequenceid=121, compaction requested=false at 1732422707675 (+6 ms)Writing region close event to WAL at 1732422707676 (+1 ms)Closed at 1732422707676
2024-11-24T04:31:47,677 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:31:47,677 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:31:47,677 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:31:47,677 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:31:47,677 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:31:47,679 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36061 is added to blk_1073741830_1006 (size=53035)
2024-11-24T04:31:47,679 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45239 is added to blk_1073741830_1006 (size=53035)
2024-11-24T04:31:47,680 INFO  [M:0;fadb2fe0ba32:35553 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-24T04:31:47,680 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-24T04:31:47,680 INFO  [M:0;fadb2fe0ba32:35553 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35553
2024-11-24T04:31:47,680 INFO  [M:0;fadb2fe0ba32:35553 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-24T04:31:47,783 INFO  [M:0;fadb2fe0ba32:35553 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-24T04:31:47,783 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35553-0x101483dca470000, quorum=127.0.0.1:49920, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-24T04:31:47,783 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35553-0x101483dca470000, quorum=127.0.0.1:49920, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-24T04:31:47,786 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@fc981fd{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T04:31:47,786 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1367dc96{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-24T04:31:47,786 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-24T04:31:47,786 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@190ad9e7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-24T04:31:47,786 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@20f59884{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b4927217-95d0-6eee-a4fe-4f3752eebc02/hadoop.log.dir/,STOPPED}
2024-11-24T04:31:47,788 WARN  [BP-946574629-172.17.0.2-1732422655191 heartbeating to localhost/127.0.0.1:35273 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-24T04:31:47,788 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-24T04:31:47,788 WARN  [BP-946574629-172.17.0.2-1732422655191 heartbeating to localhost/127.0.0.1:35273 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-946574629-172.17.0.2-1732422655191 (Datanode Uuid 10eef7a0-3f4e-4e45-91ad-1f87f08dea30) service to localhost/127.0.0.1:35273
2024-11-24T04:31:47,788 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-24T04:31:47,788 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b4927217-95d0-6eee-a4fe-4f3752eebc02/cluster_ff778f8a-035a-c4d3-e51d-256e45f57157/data/data3/current/BP-946574629-172.17.0.2-1732422655191 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T04:31:47,789 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b4927217-95d0-6eee-a4fe-4f3752eebc02/cluster_ff778f8a-035a-c4d3-e51d-256e45f57157/data/data4/current/BP-946574629-172.17.0.2-1732422655191 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T04:31:47,789 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-24T04:31:47,791 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@48bfafbe{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T04:31:47,791 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2b93cee9{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-24T04:31:47,791 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-24T04:31:47,791 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5b5b6aa1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-24T04:31:47,791 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2e6ee234{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b4927217-95d0-6eee-a4fe-4f3752eebc02/hadoop.log.dir/,STOPPED}
2024-11-24T04:31:47,793 WARN  [BP-946574629-172.17.0.2-1732422655191 heartbeating to localhost/127.0.0.1:35273 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-24T04:31:47,793 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-24T04:31:47,793 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-24T04:31:47,793 WARN  [BP-946574629-172.17.0.2-1732422655191 heartbeating to localhost/127.0.0.1:35273 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-946574629-172.17.0.2-1732422655191 (Datanode Uuid 30392bac-e63a-48f8-a36f-d0105107d846) service to localhost/127.0.0.1:35273
2024-11-24T04:31:47,793 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b4927217-95d0-6eee-a4fe-4f3752eebc02/cluster_ff778f8a-035a-c4d3-e51d-256e45f57157/data/data1/current/BP-946574629-172.17.0.2-1732422655191 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T04:31:47,794 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b4927217-95d0-6eee-a4fe-4f3752eebc02/cluster_ff778f8a-035a-c4d3-e51d-256e45f57157/data/data2/current/BP-946574629-172.17.0.2-1732422655191 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T04:31:47,794 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-24T04:31:47,800 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@62558ec9{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-24T04:31:47,800 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@294894f5{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-24T04:31:47,800 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-24T04:31:47,800 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@290f188f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-24T04:31:47,800 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@70e475c9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b4927217-95d0-6eee-a4fe-4f3752eebc02/hadoop.log.dir/,STOPPED}
2024-11-24T04:31:47,806 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-24T04:31:47,823 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-24T04:31:47,832 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=205 (was 179)
Potentially hanging thread: nioEventLoopGroup-37-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:35273
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:35273
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-12-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-34-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:35273
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-36-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-34-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-34-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-35-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-12-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-36-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:35273 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-35-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-12-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: regionserver/fadb2fe0ba32:0.leaseChecker
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.regionserver.LeaseManager.run(LeaseManager.java:82)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:35273
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-36-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-37-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:35273 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-37-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.5@localhost:35273
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-35-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:35273 from jenkins.hfs.5
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=483 (was 457) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=40 (was 103), ProcessCount=11 (was 11), AvailableMemoryMB=6464 (was 6511)
2024-11-24T04:31:47,840 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:47,840 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRolling Thread=205, OpenFileDescriptor=483, MaxFileDescriptor=1048576, SystemLoadAverage=40, ProcessCount=11, AvailableMemoryMB=6464
2024-11-24T04:31:47,840 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-24T04:31:47,840 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b4927217-95d0-6eee-a4fe-4f3752eebc02/hadoop.log.dir so I do NOT create it in target/test-data/87b7a640-4933-f8f1-d302-11da82f8fd82
2024-11-24T04:31:47,840 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b4927217-95d0-6eee-a4fe-4f3752eebc02/hadoop.tmp.dir so I do NOT create it in target/test-data/87b7a640-4933-f8f1-d302-11da82f8fd82
2024-11-24T04:31:47,841 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87b7a640-4933-f8f1-d302-11da82f8fd82/cluster_a83580c7-cecb-4f73-b131-04d66d148ce4, deleteOnExit=true
2024-11-24T04:31:47,841 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-24T04:31:47,841 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87b7a640-4933-f8f1-d302-11da82f8fd82/test.cache.data in system properties and HBase conf
2024-11-24T04:31:47,841 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:47,841 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87b7a640-4933-f8f1-d302-11da82f8fd82/hadoop.tmp.dir in system properties and HBase conf
2024-11-24T04:31:47,841 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87b7a640-4933-f8f1-d302-11da82f8fd82/hadoop.log.dir in system properties and HBase conf
2024-11-24T04:31:47,841 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87b7a640-4933-f8f1-d302-11da82f8fd82/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-24T04:31:47,841 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87b7a640-4933-f8f1-d302-11da82f8fd82/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-24T04:31:47,841 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-24T04:31:47,841 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-24T04:31:47,841 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87b7a640-4933-f8f1-d302-11da82f8fd82/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-24T04:31:47,841 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87b7a640-4933-f8f1-d302-11da82f8fd82/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-24T04:31:47,841 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87b7a640-4933-f8f1-d302-11da82f8fd82/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-24T04:31:47,841 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87b7a640-4933-f8f1-d302-11da82f8fd82/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-24T04:31:47,841 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87b7a640-4933-f8f1-d302-11da82f8fd82/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-24T04:31:47,841 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87b7a640-4933-f8f1-d302-11da82f8fd82/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-24T04:31:47,842 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87b7a640-4933-f8f1-d302-11da82f8fd82/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-24T04:31:47,842 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87b7a640-4933-f8f1-d302-11da82f8fd82/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-24T04:31:47,842 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87b7a640-4933-f8f1-d302-11da82f8fd82/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-24T04:31:47,842 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87b7a640-4933-f8f1-d302-11da82f8fd82/nfs.dump.dir in system properties and HBase conf
2024-11-24T04:31:47,842 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87b7a640-4933-f8f1-d302-11da82f8fd82/java.io.tmpdir in system properties and HBase conf
2024-11-24T04:31:47,842 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87b7a640-4933-f8f1-d302-11da82f8fd82/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-24T04:31:47,842 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87b7a640-4933-f8f1-d302-11da82f8fd82/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-24T04:31:47,842 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87b7a640-4933-f8f1-d302-11da82f8fd82/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-24T04:31:47,855 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-24T04:31:47,911 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T04:31:47,914 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-24T04:31:47,915 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-24T04:31:47,915 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-24T04:31:47,915 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-24T04:31:47,916 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T04:31:47,916 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3d320e1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87b7a640-4933-f8f1-d302-11da82f8fd82/hadoop.log.dir/,AVAILABLE}
2024-11-24T04:31:47,916 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7e466bdb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-24T04:31:48,030 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2ffc76d1{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87b7a640-4933-f8f1-d302-11da82f8fd82/java.io.tmpdir/jetty-localhost-43815-hadoop-hdfs-3_4_1-tests_jar-_-any-17644443422794582056/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-24T04:31:48,030 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@640c9e09{HTTP/1.1, (http/1.1)}{localhost:43815}
2024-11-24T04:31:48,030 INFO  [Time-limited test {}] server.Server(415): Started @237092ms
2024-11-24T04:31:48,043 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-24T04:31:48,096 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T04:31:48,098 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-24T04:31:48,099 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-24T04:31:48,099 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-24T04:31:48,099 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-24T04:31:48,099 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4a83b9f6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87b7a640-4933-f8f1-d302-11da82f8fd82/hadoop.log.dir/,AVAILABLE}
2024-11-24T04:31:48,099 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3c10c155{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-24T04:31:48,214 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@27e7702e{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87b7a640-4933-f8f1-d302-11da82f8fd82/java.io.tmpdir/jetty-localhost-34797-hadoop-hdfs-3_4_1-tests_jar-_-any-17077297256085399743/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T04:31:48,214 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@33382c80{HTTP/1.1, (http/1.1)}{localhost:34797}
2024-11-24T04:31:48,214 INFO  [Time-limited test {}] server.Server(415): Started @237275ms
2024-11-24T04:31:48,215 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-24T04:31:48,243 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T04:31:48,244 INFO  [regionserver/fadb2fe0ba32:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-24T04:31:48,264 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-24T04:31:48,265 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-24T04:31:48,265 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-24T04:31:48,265 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-24T04:31:48,265 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@444d0b71{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87b7a640-4933-f8f1-d302-11da82f8fd82/hadoop.log.dir/,AVAILABLE}
2024-11-24T04:31:48,265 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7193a060{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-24T04:31:48,340 WARN  [Thread-1956 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87b7a640-4933-f8f1-d302-11da82f8fd82/cluster_a83580c7-cecb-4f73-b131-04d66d148ce4/data/data2/current/BP-873427653-172.17.0.2-1732422707861/current, will proceed with Du for space computation calculation, 
2024-11-24T04:31:48,340 WARN  [Thread-1955 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87b7a640-4933-f8f1-d302-11da82f8fd82/cluster_a83580c7-cecb-4f73-b131-04d66d148ce4/data/data1/current/BP-873427653-172.17.0.2-1732422707861/current, will proceed with Du for space computation calculation, 
2024-11-24T04:31:48,357 WARN  [Thread-1934 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-24T04:31:48,359 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x76073eb3be10fc1f with lease ID 0xadef4fac947a2043: Processing first storage report for DS-83043b16-3924-4c1e-8d85-9a432f95c469 from datanode DatanodeRegistration(127.0.0.1:36479, datanodeUuid=6829c1c6-4bf3-412b-a59c-0bd2436c658b, infoPort=40733, infoSecurePort=0, ipcPort=40857, storageInfo=lv=-57;cid=testClusterID;nsid=1976687948;c=1732422707861)
2024-11-24T04:31:48,359 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x76073eb3be10fc1f with lease ID 0xadef4fac947a2043: from storage DS-83043b16-3924-4c1e-8d85-9a432f95c469 node DatanodeRegistration(127.0.0.1:36479, datanodeUuid=6829c1c6-4bf3-412b-a59c-0bd2436c658b, infoPort=40733, infoSecurePort=0, ipcPort=40857, storageInfo=lv=-57;cid=testClusterID;nsid=1976687948;c=1732422707861), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T04:31:48,360 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x76073eb3be10fc1f with lease ID 0xadef4fac947a2043: Processing first storage report for DS-3b3ccb3d-dcff-48b1-983c-0272a091fd3d from datanode DatanodeRegistration(127.0.0.1:36479, datanodeUuid=6829c1c6-4bf3-412b-a59c-0bd2436c658b, infoPort=40733, infoSecurePort=0, ipcPort=40857, storageInfo=lv=-57;cid=testClusterID;nsid=1976687948;c=1732422707861)
2024-11-24T04:31:48,360 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x76073eb3be10fc1f with lease ID 0xadef4fac947a2043: from storage DS-3b3ccb3d-dcff-48b1-983c-0272a091fd3d node DatanodeRegistration(127.0.0.1:36479, datanodeUuid=6829c1c6-4bf3-412b-a59c-0bd2436c658b, infoPort=40733, infoSecurePort=0, ipcPort=40857, storageInfo=lv=-57;cid=testClusterID;nsid=1976687948;c=1732422707861), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-24T04:31:48,384 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@76385d85{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87b7a640-4933-f8f1-d302-11da82f8fd82/java.io.tmpdir/jetty-localhost-44565-hadoop-hdfs-3_4_1-tests_jar-_-any-8081291076225069048/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T04:31:48,385 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@284a8a01{HTTP/1.1, (http/1.1)}{localhost:44565}
2024-11-24T04:31:48,385 INFO  [Time-limited test {}] server.Server(415): Started @237446ms
2024-11-24T04:31:48,386 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-24T04:31:48,492 WARN  [Thread-1981 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87b7a640-4933-f8f1-d302-11da82f8fd82/cluster_a83580c7-cecb-4f73-b131-04d66d148ce4/data/data3/current/BP-873427653-172.17.0.2-1732422707861/current, will proceed with Du for space computation calculation, 
2024-11-24T04:31:48,492 WARN  [Thread-1982 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87b7a640-4933-f8f1-d302-11da82f8fd82/cluster_a83580c7-cecb-4f73-b131-04d66d148ce4/data/data4/current/BP-873427653-172.17.0.2-1732422707861/current, will proceed with Du for space computation calculation, 
2024-11-24T04:31:48,508 WARN  [Thread-1970 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-24T04:31:48,510 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x16a4eaade8b47311 with lease ID 0xadef4fac947a2044: Processing first storage report for DS-e82f8249-076c-4682-b105-10a12cceb2a3 from datanode DatanodeRegistration(127.0.0.1:36545, datanodeUuid=883abd11-f372-4510-952d-7902c08af9d6, infoPort=36251, infoSecurePort=0, ipcPort=36603, storageInfo=lv=-57;cid=testClusterID;nsid=1976687948;c=1732422707861)
2024-11-24T04:31:48,510 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x16a4eaade8b47311 with lease ID 0xadef4fac947a2044: from storage DS-e82f8249-076c-4682-b105-10a12cceb2a3 node DatanodeRegistration(127.0.0.1:36545, datanodeUuid=883abd11-f372-4510-952d-7902c08af9d6, infoPort=36251, infoSecurePort=0, ipcPort=36603, storageInfo=lv=-57;cid=testClusterID;nsid=1976687948;c=1732422707861), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T04:31:48,510 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x16a4eaade8b47311 with lease ID 0xadef4fac947a2044: Processing first storage report for DS-4ab2a75a-4c89-48a9-acc3-23da232a903e from datanode DatanodeRegistration(127.0.0.1:36545, datanodeUuid=883abd11-f372-4510-952d-7902c08af9d6, infoPort=36251, infoSecurePort=0, ipcPort=36603, storageInfo=lv=-57;cid=testClusterID;nsid=1976687948;c=1732422707861)
2024-11-24T04:31:48,510 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x16a4eaade8b47311 with lease ID 0xadef4fac947a2044: from storage DS-4ab2a75a-4c89-48a9-acc3-23da232a903e node DatanodeRegistration(127.0.0.1:36545, datanodeUuid=883abd11-f372-4510-952d-7902c08af9d6, infoPort=36251, infoSecurePort=0, ipcPort=36603, storageInfo=lv=-57;cid=testClusterID;nsid=1976687948;c=1732422707861), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T04:31:48,608 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87b7a640-4933-f8f1-d302-11da82f8fd82
2024-11-24T04:31:48,610 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87b7a640-4933-f8f1-d302-11da82f8fd82/cluster_a83580c7-cecb-4f73-b131-04d66d148ce4/zookeeper_0, clientPort=64360, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87b7a640-4933-f8f1-d302-11da82f8fd82/cluster_a83580c7-cecb-4f73-b131-04d66d148ce4/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87b7a640-4933-f8f1-d302-11da82f8fd82/cluster_a83580c7-cecb-4f73-b131-04d66d148ce4/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-24T04:31:48,611 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=64360
2024-11-24T04:31:48,611 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T04:31:48,613 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T04:31:48,621 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36545 is added to blk_1073741825_1001 (size=7)
2024-11-24T04:31:48,621 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741825_1001 (size=7)
2024-11-24T04:31:48,622 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6 with version=8
2024-11-24T04:31:48,622 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/hbase-staging
2024-11-24T04:31:48,624 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/fadb2fe0ba32:0 server-side Connection retries=45
2024-11-24T04:31:48,624 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-24T04:31:48,624 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-24T04:31:48,624 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-24T04:31:48,624 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-24T04:31:48,624 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-24T04:31:48,624 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-24T04:31:48,624 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-24T04:31:48,625 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34999
2024-11-24T04:31:48,626 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:34999 connecting to ZooKeeper ensemble=127.0.0.1:64360
2024-11-24T04:31:48,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:349990x0, quorum=127.0.0.1:64360, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-24T04:31:48,632 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:34999-0x101483e98110000 connected
2024-11-24T04:31:48,646 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T04:31:48,648 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T04:31:48,649 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34999-0x101483e98110000, quorum=127.0.0.1:64360, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-24T04:31:48,650 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6, hbase.cluster.distributed=false
2024-11-24T04:31:48,651 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34999-0x101483e98110000, quorum=127.0.0.1:64360, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-24T04:31:48,651 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34999
2024-11-24T04:31:48,651 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34999
2024-11-24T04:31:48,652 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34999
2024-11-24T04:31:48,652 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34999
2024-11-24T04:31:48,652 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34999
2024-11-24T04:31:48,667 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/fadb2fe0ba32:0 server-side Connection retries=45
2024-11-24T04:31:48,667 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-24T04:31:48,667 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-24T04:31:48,668 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-24T04:31:48,668 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-24T04:31:48,668 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-24T04:31:48,668 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-24T04:31:48,668 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-24T04:31:48,668 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33299
2024-11-24T04:31:48,670 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33299 connecting to ZooKeeper ensemble=127.0.0.1:64360
2024-11-24T04:31:48,670 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T04:31:48,671 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T04:31:48,676 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:332990x0, quorum=127.0.0.1:64360, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-24T04:31:48,677 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:332990x0, quorum=127.0.0.1:64360, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-24T04:31:48,677 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33299-0x101483e98110001 connected
2024-11-24T04:31:48,677 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-24T04:31:48,677 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-24T04:31:48,678 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33299-0x101483e98110001, quorum=127.0.0.1:64360, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-24T04:31:48,679 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33299-0x101483e98110001, quorum=127.0.0.1:64360, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-24T04:31:48,679 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33299
2024-11-24T04:31:48,679 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33299
2024-11-24T04:31:48,679 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33299
2024-11-24T04:31:48,680 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33299
2024-11-24T04:31:48,680 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33299
2024-11-24T04:31:48,691 DEBUG [M:0;fadb2fe0ba32:34999 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;fadb2fe0ba32:34999
2024-11-24T04:31:48,692 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/fadb2fe0ba32,34999,1732422708624
2024-11-24T04:31:48,693 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33299-0x101483e98110001, quorum=127.0.0.1:64360, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-24T04:31:48,693 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34999-0x101483e98110000, quorum=127.0.0.1:64360, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-24T04:31:48,693 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34999-0x101483e98110000, quorum=127.0.0.1:64360, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/fadb2fe0ba32,34999,1732422708624
2024-11-24T04:31:48,695 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33299-0x101483e98110001, quorum=127.0.0.1:64360, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-24T04:31:48,695 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34999-0x101483e98110000, quorum=127.0.0.1:64360, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:31:48,695 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33299-0x101483e98110001, quorum=127.0.0.1:64360, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:31:48,695 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34999-0x101483e98110000, quorum=127.0.0.1:64360, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-24T04:31:48,696 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/fadb2fe0ba32,34999,1732422708624 from backup master directory
2024-11-24T04:31:48,697 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34999-0x101483e98110000, quorum=127.0.0.1:64360, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/fadb2fe0ba32,34999,1732422708624
2024-11-24T04:31:48,697 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33299-0x101483e98110001, quorum=127.0.0.1:64360, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-24T04:31:48,697 WARN  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-24T04:31:48,697 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34999-0x101483e98110000, quorum=127.0.0.1:64360, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-24T04:31:48,697 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=fadb2fe0ba32,34999,1732422708624
2024-11-24T04:31:48,701 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/hbase.id] with ID: 7373d0d1-f32c-441d-a2f8-ca36575e17e5
2024-11-24T04:31:48,701 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/.tmp/hbase.id
2024-11-24T04:31:48,706 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36545 is added to blk_1073741826_1002 (size=42)
2024-11-24T04:31:48,706 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741826_1002 (size=42)
2024-11-24T04:31:48,706 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/.tmp/hbase.id]:[hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/hbase.id]
2024-11-24T04:31:48,717 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T04:31:48,717 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-24T04:31:48,718 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-11-24T04:31:48,722 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33299-0x101483e98110001, quorum=127.0.0.1:64360, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:31:48,722 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34999-0x101483e98110000, quorum=127.0.0.1:64360, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:31:48,728 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36545 is added to blk_1073741827_1003 (size=196)
2024-11-24T04:31:48,729 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741827_1003 (size=196)
2024-11-24T04:31:48,729 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-24T04:31:48,730 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-24T04:31:48,730 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-24T04:31:48,736 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741828_1004 (size=1189)
2024-11-24T04:31:48,736 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36545 is added to blk_1073741828_1004 (size=1189)
2024-11-24T04:31:48,737 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/MasterData/data/master/store
2024-11-24T04:31:48,743 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36545 is added to blk_1073741829_1005 (size=34)
2024-11-24T04:31:48,744 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741829_1005 (size=34)
2024-11-24T04:31:48,744 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T04:31:48,744 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-24T04:31:48,744 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T04:31:48,744 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T04:31:48,744 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-24T04:31:48,744 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T04:31:48,744 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T04:31:48,744 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732422708744Disabling compacts and flushes for region at 1732422708744Disabling writes for close at 1732422708744Writing region close event to WAL at 1732422708744Closed at 1732422708744
2024-11-24T04:31:48,745 WARN  [master/fadb2fe0ba32:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/MasterData/data/master/store/.initializing
2024-11-24T04:31:48,745 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/MasterData/WALs/fadb2fe0ba32,34999,1732422708624
2024-11-24T04:31:48,748 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fadb2fe0ba32%2C34999%2C1732422708624, suffix=, logDir=hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/MasterData/WALs/fadb2fe0ba32,34999,1732422708624, archiveDir=hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/MasterData/oldWALs, maxLogs=10
2024-11-24T04:31:48,748 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor fadb2fe0ba32%2C34999%2C1732422708624.1732422708748
2024-11-24T04:31:48,752 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/MasterData/WALs/fadb2fe0ba32,34999,1732422708624/fadb2fe0ba32%2C34999%2C1732422708624.1732422708748
2024-11-24T04:31:48,756 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36251:36251),(127.0.0.1/127.0.0.1:40733:40733)]
2024-11-24T04:31:48,760 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-24T04:31:48,760 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T04:31:48,760 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T04:31:48,760 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T04:31:48,762 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-24T04:31:48,763 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-24T04:31:48,763 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:31:48,763 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T04:31:48,764 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-24T04:31:48,765 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-24T04:31:48,765 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:31:48,765 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-24T04:31:48,765 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-24T04:31:48,766 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-24T04:31:48,766 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:31:48,767 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-24T04:31:48,767 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-24T04:31:48,767 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-24T04:31:48,768 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:31:48,768 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-24T04:31:48,768 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T04:31:48,769 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-24T04:31:48,769 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-24T04:31:48,770 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T04:31:48,770 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T04:31:48,770 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-24T04:31:48,771 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T04:31:48,773 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-24T04:31:48,773 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=808860, jitterRate=0.028518810868263245}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-24T04:31:48,774 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1732422708761Initializing all the Stores at 1732422708761Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732422708761Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422708761Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422708761Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422708761Cleaning up temporary data from old regions at 1732422708770 (+9 ms)Region opened successfully at 1732422708774 (+4 ms)
2024-11-24T04:31:48,774 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-24T04:31:48,776 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e9fdd5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fadb2fe0ba32/172.17.0.2:0
2024-11-24T04:31:48,777 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-24T04:31:48,777 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-24T04:31:48,777 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-24T04:31:48,777 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-24T04:31:48,778 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-24T04:31:48,778 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-24T04:31:48,778 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-24T04:31:48,781 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-24T04:31:48,782 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34999-0x101483e98110000, quorum=127.0.0.1:64360, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-24T04:31:48,784 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-24T04:31:48,785 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-24T04:31:48,785 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34999-0x101483e98110000, quorum=127.0.0.1:64360, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-24T04:31:48,786 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-24T04:31:48,787 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-24T04:31:48,787 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34999-0x101483e98110000, quorum=127.0.0.1:64360, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-24T04:31:48,789 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-24T04:31:48,789 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34999-0x101483e98110000, quorum=127.0.0.1:64360, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-24T04:31:48,792 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-24T04:31:48,794 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34999-0x101483e98110000, quorum=127.0.0.1:64360, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-24T04:31:48,795 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-24T04:31:48,797 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34999-0x101483e98110000, quorum=127.0.0.1:64360, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-24T04:31:48,797 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33299-0x101483e98110001, quorum=127.0.0.1:64360, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-24T04:31:48,797 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33299-0x101483e98110001, quorum=127.0.0.1:64360, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:31:48,797 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34999-0x101483e98110000, quorum=127.0.0.1:64360, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:31:48,797 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=fadb2fe0ba32,34999,1732422708624, sessionid=0x101483e98110000, setting cluster-up flag (Was=false)
2024-11-24T04:31:48,801 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33299-0x101483e98110001, quorum=127.0.0.1:64360, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:31:48,801 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34999-0x101483e98110000, quorum=127.0.0.1:64360, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:31:48,806 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-24T04:31:48,807 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=fadb2fe0ba32,34999,1732422708624
2024-11-24T04:31:48,811 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33299-0x101483e98110001, quorum=127.0.0.1:64360, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:31:48,811 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34999-0x101483e98110000, quorum=127.0.0.1:64360, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:31:48,821 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-24T04:31:48,822 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=fadb2fe0ba32,34999,1732422708624
2024-11-24T04:31:48,824 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-24T04:31:48,825 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-24T04:31:48,825 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-24T04:31:48,826 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-24T04:31:48,826 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: fadb2fe0ba32,34999,1732422708624
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-24T04:31:48,827 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/fadb2fe0ba32:0, corePoolSize=5, maxPoolSize=5
2024-11-24T04:31:48,827 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/fadb2fe0ba32:0, corePoolSize=5, maxPoolSize=5
2024-11-24T04:31:48,827 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/fadb2fe0ba32:0, corePoolSize=5, maxPoolSize=5
2024-11-24T04:31:48,827 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/fadb2fe0ba32:0, corePoolSize=5, maxPoolSize=5
2024-11-24T04:31:48,827 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/fadb2fe0ba32:0, corePoolSize=10, maxPoolSize=10
2024-11-24T04:31:48,827 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:31:48,827 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/fadb2fe0ba32:0, corePoolSize=2, maxPoolSize=2
2024-11-24T04:31:48,827 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:31:48,828 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732422738828
2024-11-24T04:31:48,828 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-24T04:31:48,828 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-24T04:31:48,828 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-24T04:31:48,828 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-24T04:31:48,828 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-24T04:31:48,828 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-24T04:31:48,829 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-24T04:31:48,829 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-24T04:31:48,830 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:31:48,830 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-24T04:31:48,831 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-24T04:31:48,832 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-24T04:31:48,832 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-24T04:31:48,832 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-24T04:31:48,832 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-24T04:31:48,832 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-24T04:31:48,832 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/fadb2fe0ba32:0:becomeActiveMaster-HFileCleaner.large.0-1732422708832,5,FailOnTimeoutGroup]
2024-11-24T04:31:48,836 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/fadb2fe0ba32:0:becomeActiveMaster-HFileCleaner.small.0-1732422708832,5,FailOnTimeoutGroup]
2024-11-24T04:31:48,836 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-24T04:31:48,836 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-24T04:31:48,836 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-24T04:31:48,836 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-24T04:31:48,839 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741831_1007 (size=1321)
2024-11-24T04:31:48,840 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36545 is added to blk_1073741831_1007 (size=1321)
2024-11-24T04:31:48,840 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:48,840 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-24T04:31:48,841 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6
2024-11-24T04:31:48,841 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:48,848 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741832_1008 (size=32)
2024-11-24T04:31:48,848 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36545 is added to blk_1073741832_1008 (size=32)
2024-11-24T04:31:48,852 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T04:31:48,853 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-24T04:31:48,854 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-24T04:31:48,854 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:31:48,854 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T04:31:48,854 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-24T04:31:48,855 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-24T04:31:48,856 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:31:48,856 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T04:31:48,856 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-24T04:31:48,857 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-24T04:31:48,857 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:31:48,858 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T04:31:48,858 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-24T04:31:48,858 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-24T04:31:48,859 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:31:48,859 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T04:31:48,859 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-24T04:31:48,859 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/hbase/meta/1588230740
2024-11-24T04:31:48,860 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/hbase/meta/1588230740
2024-11-24T04:31:48,861 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-24T04:31:48,861 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-24T04:31:48,861 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-24T04:31:48,862 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-24T04:31:48,864 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-24T04:31:48,864 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=739832, jitterRate=-0.05925595760345459}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-24T04:31:48,865 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1732422708852Initializing all the Stores at 1732422708852Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732422708852Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732422708853 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422708853Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732422708853Cleaning up temporary data from old regions at 1732422708861 (+8 ms)Region opened successfully at 1732422708864 (+3 ms)
2024-11-24T04:31:48,865 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-24T04:31:48,865 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-24T04:31:48,865 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-24T04:31:48,865 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-24T04:31:48,865 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-24T04:31:48,865 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-24T04:31:48,865 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732422708865Disabling compacts and flushes for region at 1732422708865Disabling writes for close at 1732422708865Writing region close event to WAL at 1732422708865Closed at 1732422708865
2024-11-24T04:31:48,866 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-24T04:31:48,866 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-24T04:31:48,867 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-24T04:31:48,868 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-24T04:31:48,869 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-24T04:31:48,881 INFO  [RS:0;fadb2fe0ba32:33299 {}] regionserver.HRegionServer(746): ClusterId : 7373d0d1-f32c-441d-a2f8-ca36575e17e5
2024-11-24T04:31:48,881 DEBUG [RS:0;fadb2fe0ba32:33299 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-24T04:31:48,884 DEBUG [RS:0;fadb2fe0ba32:33299 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-24T04:31:48,884 DEBUG [RS:0;fadb2fe0ba32:33299 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-24T04:31:48,886 DEBUG [RS:0;fadb2fe0ba32:33299 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-24T04:31:48,886 DEBUG [RS:0;fadb2fe0ba32:33299 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b8d0047, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fadb2fe0ba32/172.17.0.2:0
2024-11-24T04:31:48,898 DEBUG [RS:0;fadb2fe0ba32:33299 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;fadb2fe0ba32:33299
2024-11-24T04:31:48,899 INFO  [RS:0;fadb2fe0ba32:33299 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-24T04:31:48,899 INFO  [RS:0;fadb2fe0ba32:33299 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-24T04:31:48,899 DEBUG [RS:0;fadb2fe0ba32:33299 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-24T04:31:48,899 INFO  [RS:0;fadb2fe0ba32:33299 {}] regionserver.HRegionServer(2659): reportForDuty to master=fadb2fe0ba32,34999,1732422708624 with port=33299, startcode=1732422708667
2024-11-24T04:31:48,899 DEBUG [RS:0;fadb2fe0ba32:33299 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-24T04:31:48,901 INFO  [HMaster-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48579, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.6 (auth:SIMPLE), service=RegionServerStatusService
2024-11-24T04:31:48,902 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34999 {}] master.ServerManager(363): Checking decommissioned status of RegionServer fadb2fe0ba32,33299,1732422708667
2024-11-24T04:31:48,902 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34999 {}] master.ServerManager(517): Registering regionserver=fadb2fe0ba32,33299,1732422708667
2024-11-24T04:31:48,904 DEBUG [RS:0;fadb2fe0ba32:33299 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6
2024-11-24T04:31:48,904 DEBUG [RS:0;fadb2fe0ba32:33299 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36181
2024-11-24T04:31:48,904 DEBUG [RS:0;fadb2fe0ba32:33299 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-24T04:31:48,906 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34999-0x101483e98110000, quorum=127.0.0.1:64360, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-24T04:31:48,906 DEBUG [RS:0;fadb2fe0ba32:33299 {}] zookeeper.ZKUtil(111): regionserver:33299-0x101483e98110001, quorum=127.0.0.1:64360, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/fadb2fe0ba32,33299,1732422708667
2024-11-24T04:31:48,906 WARN  [RS:0;fadb2fe0ba32:33299 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-24T04:31:48,906 INFO  [RS:0;fadb2fe0ba32:33299 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-24T04:31:48,906 DEBUG [RS:0;fadb2fe0ba32:33299 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/WALs/fadb2fe0ba32,33299,1732422708667
2024-11-24T04:31:48,906 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [fadb2fe0ba32,33299,1732422708667]
2024-11-24T04:31:48,910 INFO  [RS:0;fadb2fe0ba32:33299 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-24T04:31:48,912 INFO  [RS:0;fadb2fe0ba32:33299 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-24T04:31:48,912 INFO  [RS:0;fadb2fe0ba32:33299 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-24T04:31:48,912 INFO  [RS:0;fadb2fe0ba32:33299 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-24T04:31:48,912 INFO  [RS:0;fadb2fe0ba32:33299 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-24T04:31:48,913 INFO  [RS:0;fadb2fe0ba32:33299 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-24T04:31:48,913 INFO  [RS:0;fadb2fe0ba32:33299 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-24T04:31:48,913 DEBUG [RS:0;fadb2fe0ba32:33299 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:31:48,913 DEBUG [RS:0;fadb2fe0ba32:33299 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:31:48,913 DEBUG [RS:0;fadb2fe0ba32:33299 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:31:48,913 DEBUG [RS:0;fadb2fe0ba32:33299 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:31:48,913 DEBUG [RS:0;fadb2fe0ba32:33299 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:31:48,913 DEBUG [RS:0;fadb2fe0ba32:33299 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/fadb2fe0ba32:0, corePoolSize=2, maxPoolSize=2
2024-11-24T04:31:48,913 DEBUG [RS:0;fadb2fe0ba32:33299 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:31:48,913 DEBUG [RS:0;fadb2fe0ba32:33299 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:31:48,913 DEBUG [RS:0;fadb2fe0ba32:33299 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:31:48,913 DEBUG [RS:0;fadb2fe0ba32:33299 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:31:48,913 DEBUG [RS:0;fadb2fe0ba32:33299 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:31:48,913 DEBUG [RS:0;fadb2fe0ba32:33299 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:31:48,913 DEBUG [RS:0;fadb2fe0ba32:33299 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/fadb2fe0ba32:0, corePoolSize=3, maxPoolSize=3
2024-11-24T04:31:48,914 DEBUG [RS:0;fadb2fe0ba32:33299 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/fadb2fe0ba32:0, corePoolSize=3, maxPoolSize=3
2024-11-24T04:31:48,914 INFO  [RS:0;fadb2fe0ba32:33299 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-24T04:31:48,914 INFO  [RS:0;fadb2fe0ba32:33299 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-24T04:31:48,914 INFO  [RS:0;fadb2fe0ba32:33299 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-24T04:31:48,914 INFO  [RS:0;fadb2fe0ba32:33299 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-24T04:31:48,914 INFO  [RS:0;fadb2fe0ba32:33299 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-24T04:31:48,914 INFO  [RS:0;fadb2fe0ba32:33299 {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,33299,1732422708667-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-24T04:31:48,929 INFO  [RS:0;fadb2fe0ba32:33299 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-24T04:31:48,929 INFO  [RS:0;fadb2fe0ba32:33299 {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,33299,1732422708667-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-24T04:31:48,929 INFO  [RS:0;fadb2fe0ba32:33299 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T04:31:48,929 INFO  [RS:0;fadb2fe0ba32:33299 {}] regionserver.Replication(171): fadb2fe0ba32,33299,1732422708667 started
2024-11-24T04:31:48,944 INFO  [RS:0;fadb2fe0ba32:33299 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T04:31:48,944 INFO  [RS:0;fadb2fe0ba32:33299 {}] regionserver.HRegionServer(1482): Serving as fadb2fe0ba32,33299,1732422708667, RpcServer on fadb2fe0ba32/172.17.0.2:33299, sessionid=0x101483e98110001
2024-11-24T04:31:48,944 DEBUG [RS:0;fadb2fe0ba32:33299 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-24T04:31:48,944 DEBUG [RS:0;fadb2fe0ba32:33299 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager fadb2fe0ba32,33299,1732422708667
2024-11-24T04:31:48,944 DEBUG [RS:0;fadb2fe0ba32:33299 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fadb2fe0ba32,33299,1732422708667'
2024-11-24T04:31:48,944 DEBUG [RS:0;fadb2fe0ba32:33299 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-24T04:31:48,945 DEBUG [RS:0;fadb2fe0ba32:33299 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-24T04:31:48,945 DEBUG [RS:0;fadb2fe0ba32:33299 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-24T04:31:48,945 DEBUG [RS:0;fadb2fe0ba32:33299 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-24T04:31:48,945 DEBUG [RS:0;fadb2fe0ba32:33299 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager fadb2fe0ba32,33299,1732422708667
2024-11-24T04:31:48,945 DEBUG [RS:0;fadb2fe0ba32:33299 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fadb2fe0ba32,33299,1732422708667'
2024-11-24T04:31:48,945 DEBUG [RS:0;fadb2fe0ba32:33299 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-24T04:31:48,945 DEBUG [RS:0;fadb2fe0ba32:33299 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-24T04:31:48,946 DEBUG [RS:0;fadb2fe0ba32:33299 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-24T04:31:48,946 INFO  [RS:0;fadb2fe0ba32:33299 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-24T04:31:48,946 INFO  [RS:0;fadb2fe0ba32:33299 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-24T04:31:49,019 WARN  [fadb2fe0ba32:34999 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-11-24T04:31:49,047 INFO  [RS:0;fadb2fe0ba32:33299 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fadb2fe0ba32%2C33299%2C1732422708667, suffix=, logDir=hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/WALs/fadb2fe0ba32,33299,1732422708667, archiveDir=hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/oldWALs, maxLogs=32
2024-11-24T04:31:49,048 INFO  [RS:0;fadb2fe0ba32:33299 {}] monitor.StreamSlowMonitor(122): New stream slow monitor fadb2fe0ba32%2C33299%2C1732422708667.1732422709048
2024-11-24T04:31:49,053 INFO  [RS:0;fadb2fe0ba32:33299 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/WALs/fadb2fe0ba32,33299,1732422708667/fadb2fe0ba32%2C33299%2C1732422708667.1732422709048
2024-11-24T04:31:49,054 DEBUG [RS:0;fadb2fe0ba32:33299 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40733:40733),(127.0.0.1/127.0.0.1:36251:36251)]
2024-11-24T04:31:49,269 DEBUG [fadb2fe0ba32:34999 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-24T04:31:49,270 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=fadb2fe0ba32,33299,1732422708667
2024-11-24T04:31:49,271 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fadb2fe0ba32,33299,1732422708667, state=OPENING
2024-11-24T04:31:49,275 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-24T04:31:49,277 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33299-0x101483e98110001, quorum=127.0.0.1:64360, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:31:49,277 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34999-0x101483e98110000, quorum=127.0.0.1:64360, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:31:49,277 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-24T04:31:49,277 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-24T04:31:49,277 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-24T04:31:49,277 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=fadb2fe0ba32,33299,1732422708667}]
2024-11-24T04:31:49,430 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-24T04:31:49,432 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58055, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-24T04:31:49,435 INFO  [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-24T04:31:49,435 INFO  [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-24T04:31:49,437 INFO  [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fadb2fe0ba32%2C33299%2C1732422708667.meta, suffix=.meta, logDir=hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/WALs/fadb2fe0ba32,33299,1732422708667, archiveDir=hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/oldWALs, maxLogs=32
2024-11-24T04:31:49,437 INFO  [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor fadb2fe0ba32%2C33299%2C1732422708667.meta.1732422709437.meta
2024-11-24T04:31:49,442 INFO  [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/WALs/fadb2fe0ba32,33299,1732422708667/fadb2fe0ba32%2C33299%2C1732422708667.meta.1732422709437.meta
2024-11-24T04:31:49,446 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36251:36251),(127.0.0.1/127.0.0.1:40733:40733)]
2024-11-24T04:31:49,447 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-24T04:31:49,447 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-24T04:31:49,447 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-24T04:31:49,447 INFO  [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-24T04:31:49,447 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-24T04:31:49,447 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T04:31:49,447 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-24T04:31:49,447 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-24T04:31:49,449 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-24T04:31:49,449 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-24T04:31:49,449 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:31:49,450 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T04:31:49,450 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-24T04:31:49,450 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-24T04:31:49,450 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:31:49,451 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T04:31:49,451 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-24T04:31:49,451 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-24T04:31:49,451 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:31:49,452 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T04:31:49,452 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-24T04:31:49,452 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-24T04:31:49,452 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:31:49,453 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T04:31:49,453 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-24T04:31:49,453 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/hbase/meta/1588230740
2024-11-24T04:31:49,454 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/hbase/meta/1588230740
2024-11-24T04:31:49,455 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-24T04:31:49,455 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-24T04:31:49,456 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-24T04:31:49,457 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-24T04:31:49,457 INFO  [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=734786, jitterRate=-0.06567157804965973}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-24T04:31:49,457 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-24T04:31:49,458 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1732422709447Writing region info on filesystem at 1732422709447Initializing all the Stores at 1732422709448 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732422709448Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732422709448Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422709448Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732422709448Cleaning up temporary data from old regions at 1732422709455 (+7 ms)Running coprocessor post-open hooks at 1732422709457 (+2 ms)Region opened successfully at 1732422709458 (+1 ms)
2024-11-24T04:31:49,459 INFO  [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732422709430
2024-11-24T04:31:49,461 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-24T04:31:49,461 INFO  [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-24T04:31:49,462 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=fadb2fe0ba32,33299,1732422708667
2024-11-24T04:31:49,463 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fadb2fe0ba32,33299,1732422708667, state=OPEN
2024-11-24T04:31:49,467 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33299-0x101483e98110001, quorum=127.0.0.1:64360, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-24T04:31:49,467 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34999-0x101483e98110000, quorum=127.0.0.1:64360, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-24T04:31:49,467 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=fadb2fe0ba32,33299,1732422708667
2024-11-24T04:31:49,467 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-24T04:31:49,467 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-24T04:31:49,470 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-24T04:31:49,470 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fadb2fe0ba32,33299,1732422708667 in 190 msec
2024-11-24T04:31:49,472 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-24T04:31:49,472 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 604 msec
2024-11-24T04:31:49,473 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-24T04:31:49,473 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-24T04:31:49,474 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-24T04:31:49,474 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fadb2fe0ba32,33299,1732422708667, seqNum=-1]
2024-11-24T04:31:49,474 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-24T04:31:49,475 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-15-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42515, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-24T04:31:49,480 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 654 msec
2024-11-24T04:31:49,480 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732422709480, completionTime=-1
2024-11-24T04:31:49,480 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-24T04:31:49,480 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-24T04:31:49,482 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-24T04:31:49,482 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732422769482
2024-11-24T04:31:49,482 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732422829482
2024-11-24T04:31:49,482 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 1 msec
2024-11-24T04:31:49,482 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,34999,1732422708624-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-24T04:31:49,482 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,34999,1732422708624-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T04:31:49,482 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,34999,1732422708624-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T04:31:49,482 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-fadb2fe0ba32:34999, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T04:31:49,482 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-24T04:31:49,482 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-24T04:31:49,484 DEBUG [master/fadb2fe0ba32:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-24T04:31:49,485 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.788sec
2024-11-24T04:31:49,485 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-24T04:31:49,485 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-24T04:31:49,485 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-24T04:31:49,485 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-24T04:31:49,485 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-24T04:31:49,485 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,34999,1732422708624-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-24T04:31:49,485 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,34999,1732422708624-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-24T04:31:49,487 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-24T04:31:49,487 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-24T04:31:49,487 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,34999,1732422708624-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T04:31:49,582 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47c8059, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-24T04:31:49,582 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fadb2fe0ba32,34999,-1 for getting cluster id
2024-11-24T04:31:49,582 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-24T04:31:49,583 DEBUG [HMaster-EventLoopGroup-14-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7373d0d1-f32c-441d-a2f8-ca36575e17e5'
2024-11-24T04:31:49,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-24T04:31:49,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7373d0d1-f32c-441d-a2f8-ca36575e17e5"

2024-11-24T04:31:49,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19e7b7bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-24T04:31:49,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fadb2fe0ba32,34999,-1]
2024-11-24T04:31:49,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-24T04:31:49,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T04:31:49,585 INFO  [HMaster-EventLoopGroup-14-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33448, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-24T04:31:49,586 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49e87d3f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-24T04:31:49,586 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-24T04:31:49,587 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fadb2fe0ba32,33299,1732422708667, seqNum=-1]
2024-11-24T04:31:49,587 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-24T04:31:49,588 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46948, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-24T04:31:49,590 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=fadb2fe0ba32,34999,1732422708624
2024-11-24T04:31:49,590 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T04:31:49,592 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-24T04:31:49,592 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-24T04:31:49,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is fadb2fe0ba32,34999,1732422708624
2024-11-24T04:31:49,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2195603a
2024-11-24T04:31:49,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-24T04:31:49,594 INFO  [HMaster-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33462, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-24T04:31:49,595 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34999 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-24T04:31:49,595 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34999 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-24T04:31:49,595 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34999 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-24T04:31:49,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34999 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRolling
2024-11-24T04:31:49,597 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-11-24T04:31:49,597 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:31:49,597 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34999 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRolling"
 procId is: 4
2024-11-24T04:31:49,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34999 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-24T04:31:49,598 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-24T04:31:49,604 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36545 is added to blk_1073741835_1011 (size=381)
2024-11-24T04:31:49,604 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741835_1011 (size=381)
2024-11-24T04:31:49,606 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a96be4dd9305efb471150cc3d77a7019, NAME => 'TestLogRolling-testLogRolling,,1732422709594.a96be4dd9305efb471150cc3d77a7019.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6
2024-11-24T04:31:49,611 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741836_1012 (size=64)
2024-11-24T04:31:49,611 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36545 is added to blk_1073741836_1012 (size=64)
2024-11-24T04:31:49,612 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1732422709594.a96be4dd9305efb471150cc3d77a7019.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T04:31:49,612 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1722): Closing a96be4dd9305efb471150cc3d77a7019, disabling compactions & flushes
2024-11-24T04:31:49,612 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1732422709594.a96be4dd9305efb471150cc3d77a7019.
2024-11-24T04:31:49,612 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1732422709594.a96be4dd9305efb471150cc3d77a7019.
2024-11-24T04:31:49,612 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1732422709594.a96be4dd9305efb471150cc3d77a7019. after waiting 0 ms
2024-11-24T04:31:49,612 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1732422709594.a96be4dd9305efb471150cc3d77a7019.
2024-11-24T04:31:49,612 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1732422709594.a96be4dd9305efb471150cc3d77a7019.
2024-11-24T04:31:49,612 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for a96be4dd9305efb471150cc3d77a7019:
Waiting for close lock at 1732422709612Disabling compacts and flushes for region at 1732422709612Disabling writes for close at 1732422709612Writing region close event to WAL at 1732422709612Closed at 1732422709612
2024-11-24T04:31:49,613 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ADD_TO_META
2024-11-24T04:31:49,613 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRolling,,1732422709594.a96be4dd9305efb471150cc3d77a7019.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1732422709613"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732422709613"}]},"ts":"1732422709613"}
2024-11-24T04:31:49,615 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-24T04:31:49,616 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-24T04:31:49,617 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732422709616"}]},"ts":"1732422709616"}
2024-11-24T04:31:49,618 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRolling, state=ENABLING in hbase:meta
2024-11-24T04:31:49,619 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=a96be4dd9305efb471150cc3d77a7019, ASSIGN}]
2024-11-24T04:31:49,620 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=a96be4dd9305efb471150cc3d77a7019, ASSIGN
2024-11-24T04:31:49,621 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=a96be4dd9305efb471150cc3d77a7019, ASSIGN; state=OFFLINE, location=fadb2fe0ba32,33299,1732422708667; forceNewPlan=false, retain=false
2024-11-24T04:31:49,771 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=a96be4dd9305efb471150cc3d77a7019, regionState=OPENING, regionLocation=fadb2fe0ba32,33299,1732422708667
2024-11-24T04:31:49,774 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=a96be4dd9305efb471150cc3d77a7019, ASSIGN because future has completed
2024-11-24T04:31:49,774 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure a96be4dd9305efb471150cc3d77a7019, server=fadb2fe0ba32,33299,1732422708667}]
2024-11-24T04:31:49,841 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:49,841 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:49,931 INFO  [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,,1732422709594.a96be4dd9305efb471150cc3d77a7019.
2024-11-24T04:31:49,931 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => a96be4dd9305efb471150cc3d77a7019, NAME => 'TestLogRolling-testLogRolling,,1732422709594.a96be4dd9305efb471150cc3d77a7019.', STARTKEY => '', ENDKEY => ''}
2024-11-24T04:31:49,931 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling a96be4dd9305efb471150cc3d77a7019
2024-11-24T04:31:49,931 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1732422709594.a96be4dd9305efb471150cc3d77a7019.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T04:31:49,931 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for a96be4dd9305efb471150cc3d77a7019
2024-11-24T04:31:49,931 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for a96be4dd9305efb471150cc3d77a7019
2024-11-24T04:31:49,933 INFO  [StoreOpener-a96be4dd9305efb471150cc3d77a7019-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region a96be4dd9305efb471150cc3d77a7019 
2024-11-24T04:31:49,934 INFO  [StoreOpener-a96be4dd9305efb471150cc3d77a7019-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a96be4dd9305efb471150cc3d77a7019 columnFamilyName info
2024-11-24T04:31:49,934 DEBUG [StoreOpener-a96be4dd9305efb471150cc3d77a7019-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:31:49,934 INFO  [StoreOpener-a96be4dd9305efb471150cc3d77a7019-1 {}] regionserver.HStore(327): Store=a96be4dd9305efb471150cc3d77a7019/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-24T04:31:49,934 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for a96be4dd9305efb471150cc3d77a7019
2024-11-24T04:31:49,935 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019
2024-11-24T04:31:49,935 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019
2024-11-24T04:31:49,935 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for a96be4dd9305efb471150cc3d77a7019
2024-11-24T04:31:49,935 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for a96be4dd9305efb471150cc3d77a7019
2024-11-24T04:31:49,937 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for a96be4dd9305efb471150cc3d77a7019
2024-11-24T04:31:49,938 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-24T04:31:49,939 INFO  [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened a96be4dd9305efb471150cc3d77a7019; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=828409, jitterRate=0.053376585245132446}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-24T04:31:49,939 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a96be4dd9305efb471150cc3d77a7019
2024-11-24T04:31:49,939 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for a96be4dd9305efb471150cc3d77a7019:
Running coprocessor pre-open hook at 1732422709932Writing region info on filesystem at 1732422709932Initializing all the Stores at 1732422709932Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422709932Cleaning up temporary data from old regions at 1732422709935 (+3 ms)Running coprocessor post-open hooks at 1732422709939 (+4 ms)Region opened successfully at 1732422709939
2024-11-24T04:31:49,940 INFO  [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,,1732422709594.a96be4dd9305efb471150cc3d77a7019., pid=6, masterSystemTime=1732422709927
2024-11-24T04:31:49,942 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,,1732422709594.a96be4dd9305efb471150cc3d77a7019.
2024-11-24T04:31:49,942 INFO  [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,,1732422709594.a96be4dd9305efb471150cc3d77a7019.
2024-11-24T04:31:49,943 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=a96be4dd9305efb471150cc3d77a7019, regionState=OPEN, openSeqNum=2, regionLocation=fadb2fe0ba32,33299,1732422708667
2024-11-24T04:31:49,945 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure a96be4dd9305efb471150cc3d77a7019, server=fadb2fe0ba32,33299,1732422708667 because future has completed
2024-11-24T04:31:49,949 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-24T04:31:49,949 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure a96be4dd9305efb471150cc3d77a7019, server=fadb2fe0ba32,33299,1732422708667 in 173 msec
2024-11-24T04:31:49,951 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-24T04:31:49,951 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=a96be4dd9305efb471150cc3d77a7019, ASSIGN in 331 msec
2024-11-24T04:31:49,952 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-24T04:31:49,952 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732422709952"}]},"ts":"1732422709952"}
2024-11-24T04:31:49,954 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRolling, state=ENABLED in hbase:meta
2024-11-24T04:31:49,955 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_POST_OPERATION
2024-11-24T04:31:49,957 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRolling in 360 msec
2024-11-24T04:31:50,841 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:50,842 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:51,842 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:51,842 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:52,373 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:31:52,373 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:31:52,374 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:31:52,374 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:31:52,374 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:31:52,374 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:31:52,374 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:31:52,375 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:31:52,387 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:31:52,387 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:31:52,387 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:31:52,387 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:31:52,387 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:31:52,387 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:31:52,390 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:31:52,390 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:31:52,390 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:31:52,392 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:31:52,842 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:52,842 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:52,896 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-24T04:31:52,897 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:31:52,897 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:31:52,898 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:31:52,898 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:31:52,898 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:31:52,898 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:31:52,899 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:31:52,899 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:31:52,914 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:31:52,915 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:31:52,915 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:31:52,915 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:31:52,915 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:31:52,915 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:31:52,918 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:31:52,919 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:31:52,919 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:31:52,921 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:31:53,843 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:53,843 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:54,843 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:54,843 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:54,910 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-24T04:31:54,910 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRolling'
2024-11-24T04:31:55,844 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:55,844 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:56,266 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling
2024-11-24T04:31:56,266 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling Metrics about Tables on a single HBase RegionServer
2024-11-24T04:31:56,267 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-24T04:31:56,845 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:56,845 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:57,845 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:57,845 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:58,846 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:58,846 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:59,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34999 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-24T04:31:59,673 INFO  [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRolling completed
2024-11-24T04:31:59,673 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRolling,, stopping at row=TestLogRolling-testLogRolling ,, for max=2147483647 with caching=100
2024-11-24T04:31:59,676 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRolling
2024-11-24T04:31:59,676 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRolling,,1732422709594.a96be4dd9305efb471150cc3d77a7019.
2024-11-24T04:31:59,679 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testLogRolling,,1732422709594.a96be4dd9305efb471150cc3d77a7019., hostname=fadb2fe0ba32,33299,1732422708667, seqNum=2]
2024-11-24T04:31:59,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33299 {}] regionserver.HRegion(8855): Flush requested on a96be4dd9305efb471150cc3d77a7019
2024-11-24T04:31:59,690 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a96be4dd9305efb471150cc3d77a7019 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-24T04:31:59,706 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/.tmp/info/57b6bc47fbde491780522b406a9e3c58 is 1080, key is row0001/info:/1732422719680/Put/seqid=0
2024-11-24T04:31:59,711 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741837_1013 (size=12509)
2024-11-24T04:31:59,711 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36545 is added to blk_1073741837_1013 (size=12509)
2024-11-24T04:31:59,712 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/.tmp/info/57b6bc47fbde491780522b406a9e3c58
2024-11-24T04:31:59,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/.tmp/info/57b6bc47fbde491780522b406a9e3c58 as hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/info/57b6bc47fbde491780522b406a9e3c58
2024-11-24T04:31:59,725 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/info/57b6bc47fbde491780522b406a9e3c58, entries=7, sequenceid=11, filesize=12.2 K
2024-11-24T04:31:59,726 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=22.07 KB/22596 for a96be4dd9305efb471150cc3d77a7019 in 36ms, sequenceid=11, compaction requested=false
2024-11-24T04:31:59,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a96be4dd9305efb471150cc3d77a7019:

2024-11-24T04:31:59,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33299 {}] regionserver.HRegion(8855): Flush requested on a96be4dd9305efb471150cc3d77a7019
2024-11-24T04:31:59,727 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a96be4dd9305efb471150cc3d77a7019 1/1 column families, dataSize=23.12 KB heapSize=25 KB
2024-11-24T04:31:59,732 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/.tmp/info/787d096725d842a2bf9c552c8218e92f is 1080, key is row0008/info:/1732422719691/Put/seqid=0
2024-11-24T04:31:59,737 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36545 is added to blk_1073741838_1014 (size=28684)
2024-11-24T04:31:59,738 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741838_1014 (size=28684)
2024-11-24T04:31:59,738 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=23.12 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/.tmp/info/787d096725d842a2bf9c552c8218e92f
2024-11-24T04:31:59,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/.tmp/info/787d096725d842a2bf9c552c8218e92f as hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/info/787d096725d842a2bf9c552c8218e92f
2024-11-24T04:31:59,748 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/info/787d096725d842a2bf9c552c8218e92f, entries=22, sequenceid=36, filesize=28.0 K
2024-11-24T04:31:59,749 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.12 KB/23672, heapSize ~24.98 KB/25584, currentSize=3.15 KB/3228 for a96be4dd9305efb471150cc3d77a7019 in 22ms, sequenceid=36, compaction requested=false
2024-11-24T04:31:59,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a96be4dd9305efb471150cc3d77a7019:

2024-11-24T04:31:59,750 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=40.2 K, sizeToCheck=16.0 K
2024-11-24T04:31:59,750 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-24T04:31:59,750 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/info/787d096725d842a2bf9c552c8218e92f because midkey is the same as first or last row
2024-11-24T04:31:59,847 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:31:59,847 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:00,847 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:00,847 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:01,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33299 {}] regionserver.HRegion(8855): Flush requested on a96be4dd9305efb471150cc3d77a7019
2024-11-24T04:32:01,741 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a96be4dd9305efb471150cc3d77a7019 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-24T04:32:01,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/.tmp/info/4656bb4ec4e140c482919c3942ecbedf is 1080, key is row0030/info:/1732422719728/Put/seqid=0
2024-11-24T04:32:01,754 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741839_1015 (size=12509)
2024-11-24T04:32:01,754 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36545 is added to blk_1073741839_1015 (size=12509)
2024-11-24T04:32:01,754 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/.tmp/info/4656bb4ec4e140c482919c3942ecbedf
2024-11-24T04:32:01,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/.tmp/info/4656bb4ec4e140c482919c3942ecbedf as hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/info/4656bb4ec4e140c482919c3942ecbedf
2024-11-24T04:32:01,766 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/info/4656bb4ec4e140c482919c3942ecbedf, entries=7, sequenceid=46, filesize=12.2 K
2024-11-24T04:32:01,767 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=15.76 KB/16140 for a96be4dd9305efb471150cc3d77a7019 in 26ms, sequenceid=46, compaction requested=true
2024-11-24T04:32:01,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a96be4dd9305efb471150cc3d77a7019:

2024-11-24T04:32:01,768 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=52.4 K, sizeToCheck=16.0 K
2024-11-24T04:32:01,768 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-24T04:32:01,768 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/info/787d096725d842a2bf9c552c8218e92f because midkey is the same as first or last row
2024-11-24T04:32:01,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a96be4dd9305efb471150cc3d77a7019:info, priority=-2147483648, current under compaction store size is 1
2024-11-24T04:32:01,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-24T04:32:01,768 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-24T04:32:01,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33299 {}] regionserver.HRegion(8855): Flush requested on a96be4dd9305efb471150cc3d77a7019
2024-11-24T04:32:01,769 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a96be4dd9305efb471150cc3d77a7019 1/1 column families, dataSize=17.86 KB heapSize=19.38 KB
2024-11-24T04:32:01,769 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-24T04:32:01,769 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 53702 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-24T04:32:01,769 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.HStore(1541): a96be4dd9305efb471150cc3d77a7019/info is initiating minor compaction (all files)
2024-11-24T04:32:01,769 INFO  [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a96be4dd9305efb471150cc3d77a7019/info in TestLogRolling-testLogRolling,,1732422709594.a96be4dd9305efb471150cc3d77a7019.
2024-11-24T04:32:01,770 INFO  [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/info/57b6bc47fbde491780522b406a9e3c58, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/info/787d096725d842a2bf9c552c8218e92f, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/info/4656bb4ec4e140c482919c3942ecbedf] into tmpdir=hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/.tmp, totalSize=52.4 K
2024-11-24T04:32:01,770 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57b6bc47fbde491780522b406a9e3c58, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732422719680
2024-11-24T04:32:01,770 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:32:01,770 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:32:01,771 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 787d096725d842a2bf9c552c8218e92f, keycount=22, bloomtype=ROW, size=28.0 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1732422719691
2024-11-24T04:32:01,771 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:32:01,771 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:32:01,771 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4656bb4ec4e140c482919c3942ecbedf, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732422719728
2024-11-24T04:32:01,771 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:32:01,771 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:32:01,772 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:32:01,772 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:32:01,773 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/.tmp/info/f3357cfa50ae4c8dabdb2037a0666cb1 is 1080, key is row0037/info:/1732422721742/Put/seqid=0
2024-11-24T04:32:01,785 INFO  [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a96be4dd9305efb471150cc3d77a7019#info#compaction#59 average throughput is 18.47 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-24T04:32:01,785 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36545 is added to blk_1073741840_1016 (size=23299)
2024-11-24T04:32:01,785 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/.tmp/info/9f2c9ed2f9c0463481f67370f6b7cd0e is 1080, key is row0001/info:/1732422719680/Put/seqid=0
2024-11-24T04:32:01,786 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741840_1016 (size=23299)
2024-11-24T04:32:01,786 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.86 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/.tmp/info/f3357cfa50ae4c8dabdb2037a0666cb1
2024-11-24T04:32:01,791 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741841_1017 (size=43901)
2024-11-24T04:32:01,792 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36545 is added to blk_1073741841_1017 (size=43901)
2024-11-24T04:32:01,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/.tmp/info/f3357cfa50ae4c8dabdb2037a0666cb1 as hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/info/f3357cfa50ae4c8dabdb2037a0666cb1
2024-11-24T04:32:01,797 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:32:01,797 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:32:01,798 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:32:01,798 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/info/f3357cfa50ae4c8dabdb2037a0666cb1, entries=17, sequenceid=66, filesize=22.8 K
2024-11-24T04:32:01,798 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:32:01,798 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/.tmp/info/9f2c9ed2f9c0463481f67370f6b7cd0e as hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/info/9f2c9ed2f9c0463481f67370f6b7cd0e
2024-11-24T04:32:01,798 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:32:01,799 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:32:01,800 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~17.86 KB/18292, heapSize ~19.36 KB/19824, currentSize=11.56 KB/11836 for a96be4dd9305efb471150cc3d77a7019 in 30ms, sequenceid=66, compaction requested=false
2024-11-24T04:32:01,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a96be4dd9305efb471150cc3d77a7019:

2024-11-24T04:32:01,800 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=75.2 K, sizeToCheck=16.0 K
2024-11-24T04:32:01,800 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-24T04:32:01,800 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/info/787d096725d842a2bf9c552c8218e92f because midkey is the same as first or last row
2024-11-24T04:32:01,802 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:32:01,803 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:32:01,803 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:32:01,805 INFO  [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a96be4dd9305efb471150cc3d77a7019/info of a96be4dd9305efb471150cc3d77a7019 into 9f2c9ed2f9c0463481f67370f6b7cd0e(size=42.9 K), total size for store is 65.6 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-24T04:32:01,805 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a96be4dd9305efb471150cc3d77a7019:

2024-11-24T04:32:01,805 INFO  [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1732422709594.a96be4dd9305efb471150cc3d77a7019., storeName=a96be4dd9305efb471150cc3d77a7019/info, priority=13, startTime=1732422721768; duration=0sec
2024-11-24T04:32:01,805 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=65.6 K, sizeToCheck=16.0 K
2024-11-24T04:32:01,805 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-24T04:32:01,805 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/info/9f2c9ed2f9c0463481f67370f6b7cd0e because midkey is the same as first or last row
2024-11-24T04:32:01,805 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=65.6 K, sizeToCheck=16.0 K
2024-11-24T04:32:01,805 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-24T04:32:01,805 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/info/9f2c9ed2f9c0463481f67370f6b7cd0e because midkey is the same as first or last row
2024-11-24T04:32:01,805 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=65.6 K, sizeToCheck=16.0 K
2024-11-24T04:32:01,805 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-24T04:32:01,805 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/info/9f2c9ed2f9c0463481f67370f6b7cd0e because midkey is the same as first or last row
2024-11-24T04:32:01,806 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-24T04:32:01,806 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a96be4dd9305efb471150cc3d77a7019:info
2024-11-24T04:32:01,806 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:32:01,848 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:01,848 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:02,848 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:02,848 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:03,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33299 {}] regionserver.HRegion(8855): Flush requested on a96be4dd9305efb471150cc3d77a7019
2024-11-24T04:32:03,790 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a96be4dd9305efb471150cc3d77a7019 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-11-24T04:32:03,795 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/.tmp/info/da651c9fe3ae4ba894afc503b6b1343d is 1080, key is row0054/info:/1732422721770/Put/seqid=0
2024-11-24T04:32:03,800 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36545 is added to blk_1073741842_1018 (size=17894)
2024-11-24T04:32:03,801 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741842_1018 (size=17894)
2024-11-24T04:32:03,801 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/.tmp/info/da651c9fe3ae4ba894afc503b6b1343d
2024-11-24T04:32:03,807 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/.tmp/info/da651c9fe3ae4ba894afc503b6b1343d as hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/info/da651c9fe3ae4ba894afc503b6b1343d
2024-11-24T04:32:03,812 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/info/da651c9fe3ae4ba894afc503b6b1343d, entries=12, sequenceid=82, filesize=17.5 K
2024-11-24T04:32:03,813 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=13.66 KB/13988 for a96be4dd9305efb471150cc3d77a7019 in 23ms, sequenceid=82, compaction requested=true
2024-11-24T04:32:03,813 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a96be4dd9305efb471150cc3d77a7019:

2024-11-24T04:32:03,813 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=83.1 K, sizeToCheck=16.0 K
2024-11-24T04:32:03,813 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-24T04:32:03,813 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/info/9f2c9ed2f9c0463481f67370f6b7cd0e because midkey is the same as first or last row
2024-11-24T04:32:03,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a96be4dd9305efb471150cc3d77a7019:info, priority=-2147483648, current under compaction store size is 1
2024-11-24T04:32:03,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-24T04:32:03,814 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-24T04:32:03,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33299 {}] regionserver.HRegion(8855): Flush requested on a96be4dd9305efb471150cc3d77a7019
2024-11-24T04:32:03,814 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a96be4dd9305efb471150cc3d77a7019 1/1 column families, dataSize=14.71 KB heapSize=16 KB
2024-11-24T04:32:03,815 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 85094 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-24T04:32:03,815 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.HStore(1541): a96be4dd9305efb471150cc3d77a7019/info is initiating minor compaction (all files)
2024-11-24T04:32:03,815 INFO  [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a96be4dd9305efb471150cc3d77a7019/info in TestLogRolling-testLogRolling,,1732422709594.a96be4dd9305efb471150cc3d77a7019.
2024-11-24T04:32:03,815 INFO  [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/info/9f2c9ed2f9c0463481f67370f6b7cd0e, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/info/f3357cfa50ae4c8dabdb2037a0666cb1, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/info/da651c9fe3ae4ba894afc503b6b1343d] into tmpdir=hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/.tmp, totalSize=83.1 K
2024-11-24T04:32:03,816 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f2c9ed2f9c0463481f67370f6b7cd0e, keycount=36, bloomtype=ROW, size=42.9 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732422719680
2024-11-24T04:32:03,816 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3357cfa50ae4c8dabdb2037a0666cb1, keycount=17, bloomtype=ROW, size=22.8 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1732422721742
2024-11-24T04:32:03,817 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] compactions.Compactor(225): Compacting da651c9fe3ae4ba894afc503b6b1343d, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732422721770
2024-11-24T04:32:03,818 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/.tmp/info/6df4256bb0f1422cac4b53f1647f7eaa is 1080, key is row0066/info:/1732422723792/Put/seqid=0
2024-11-24T04:32:03,824 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36545 is added to blk_1073741843_1019 (size=20064)
2024-11-24T04:32:03,824 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741843_1019 (size=20064)
2024-11-24T04:32:03,828 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.71 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/.tmp/info/6df4256bb0f1422cac4b53f1647f7eaa
2024-11-24T04:32:03,832 INFO  [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a96be4dd9305efb471150cc3d77a7019#info#compaction#62 average throughput is 22.23 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-24T04:32:03,833 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/.tmp/info/005ca3e2be6042eab9bd6f6c772b9469 is 1080, key is row0001/info:/1732422719680/Put/seqid=0
2024-11-24T04:32:03,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/.tmp/info/6df4256bb0f1422cac4b53f1647f7eaa as hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/info/6df4256bb0f1422cac4b53f1647f7eaa
2024-11-24T04:32:03,836 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36545 is added to blk_1073741844_1020 (size=75378)
2024-11-24T04:32:03,836 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741844_1020 (size=75378)
2024-11-24T04:32:03,842 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/info/6df4256bb0f1422cac4b53f1647f7eaa, entries=14, sequenceid=99, filesize=19.6 K
2024-11-24T04:32:03,843 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~14.71 KB/15064, heapSize ~15.98 KB/16368, currentSize=14.71 KB/15064 for a96be4dd9305efb471150cc3d77a7019 in 29ms, sequenceid=99, compaction requested=false
2024-11-24T04:32:03,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a96be4dd9305efb471150cc3d77a7019:

2024-11-24T04:32:03,843 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=102.7 K, sizeToCheck=16.0 K
2024-11-24T04:32:03,843 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-24T04:32:03,843 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/info/9f2c9ed2f9c0463481f67370f6b7cd0e because midkey is the same as first or last row
2024-11-24T04:32:03,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33299 {}] regionserver.HRegion(8855): Flush requested on a96be4dd9305efb471150cc3d77a7019
2024-11-24T04:32:03,843 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a96be4dd9305efb471150cc3d77a7019 1/1 column families, dataSize=15.76 KB heapSize=17.13 KB
2024-11-24T04:32:03,843 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/.tmp/info/005ca3e2be6042eab9bd6f6c772b9469 as hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/info/005ca3e2be6042eab9bd6f6c772b9469
2024-11-24T04:32:03,848 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/.tmp/info/dda9939d8dc9423abdc85e4d4095a78f is 1080, key is row0080/info:/1732422723815/Put/seqid=0
2024-11-24T04:32:03,849 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:03,849 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:03,850 INFO  [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a96be4dd9305efb471150cc3d77a7019/info of a96be4dd9305efb471150cc3d77a7019 into 005ca3e2be6042eab9bd6f6c772b9469(size=73.6 K), total size for store is 93.2 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-24T04:32:03,850 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a96be4dd9305efb471150cc3d77a7019:

2024-11-24T04:32:03,850 INFO  [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1732422709594.a96be4dd9305efb471150cc3d77a7019., storeName=a96be4dd9305efb471150cc3d77a7019/info, priority=13, startTime=1732422723813; duration=0sec
2024-11-24T04:32:03,850 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=93.2 K, sizeToCheck=16.0 K
2024-11-24T04:32:03,850 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-24T04:32:03,850 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=93.2 K, sizeToCheck=16.0 K
2024-11-24T04:32:03,850 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-24T04:32:03,850 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=93.2 K, sizeToCheck=16.0 K
2024-11-24T04:32:03,850 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-24T04:32:03,853 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.CompactSplit(239): Splitting TestLogRolling-testLogRolling,,1732422709594.a96be4dd9305efb471150cc3d77a7019., compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-24T04:32:03,853 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-24T04:32:03,853 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a96be4dd9305efb471150cc3d77a7019:info
2024-11-24T04:32:03,854 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36545 is added to blk_1073741845_1021 (size=21141)
2024-11-24T04:32:03,854 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34999 {}] assignment.AssignmentManager(1363): Split request from fadb2fe0ba32,33299,1732422708667, parent={ENCODED => a96be4dd9305efb471150cc3d77a7019, NAME => 'TestLogRolling-testLogRolling,,1732422709594.a96be4dd9305efb471150cc3d77a7019.', STARTKEY => '', ENDKEY => ''}, splitKey=row0062
2024-11-24T04:32:03,854 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741845_1021 (size=21141)
2024-11-24T04:32:03,855 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.76 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/.tmp/info/dda9939d8dc9423abdc85e4d4095a78f
2024-11-24T04:32:03,860 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34999 {}] assignment.SplitTableRegionProcedure(223): Splittable=true state=OPEN, location=fadb2fe0ba32,33299,1732422708667
2024-11-24T04:32:03,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/.tmp/info/dda9939d8dc9423abdc85e4d4095a78f as hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/info/dda9939d8dc9423abdc85e4d4095a78f
2024-11-24T04:32:03,865 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34999 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=a96be4dd9305efb471150cc3d77a7019, daughterA=9449b123c00b1a221a275bb41283b414, daughterB=3f22261b1394925c40b35fb57958000b
2024-11-24T04:32:03,866 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=a96be4dd9305efb471150cc3d77a7019, daughterA=9449b123c00b1a221a275bb41283b414, daughterB=3f22261b1394925c40b35fb57958000b
2024-11-24T04:32:03,866 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=a96be4dd9305efb471150cc3d77a7019, daughterA=9449b123c00b1a221a275bb41283b414, daughterB=3f22261b1394925c40b35fb57958000b
2024-11-24T04:32:03,866 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=a96be4dd9305efb471150cc3d77a7019, daughterA=9449b123c00b1a221a275bb41283b414, daughterB=3f22261b1394925c40b35fb57958000b
2024-11-24T04:32:03,868 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/info/dda9939d8dc9423abdc85e4d4095a78f, entries=15, sequenceid=117, filesize=20.6 K
2024-11-24T04:32:03,869 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~15.76 KB/16140, heapSize ~17.11 KB/17520, currentSize=2.10 KB/2152 for a96be4dd9305efb471150cc3d77a7019 in 26ms, sequenceid=117, compaction requested=true
2024-11-24T04:32:03,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a96be4dd9305efb471150cc3d77a7019:

2024-11-24T04:32:03,870 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=113.9 K, sizeToCheck=16.0 K
2024-11-24T04:32:03,870 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-24T04:32:03,870 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=113.9 K, sizeToCheck=16.0 K
2024-11-24T04:32:03,870 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-24T04:32:03,870 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=113.9 K, sizeToCheck=16.0 K
2024-11-24T04:32:03,870 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-24T04:32:03,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(239): Splitting TestLogRolling-testLogRolling,,1732422709594.a96be4dd9305efb471150cc3d77a7019., compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-24T04:32:03,873 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34999 {}] assignment.AssignmentManager(1363): Split request from fadb2fe0ba32,33299,1732422708667, parent={ENCODED => a96be4dd9305efb471150cc3d77a7019, NAME => 'TestLogRolling-testLogRolling,,1732422709594.a96be4dd9305efb471150cc3d77a7019.', STARTKEY => '', ENDKEY => ''}, splitKey=row0062
2024-11-24T04:32:03,873 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34999 {}] assignment.AssignmentManager(1378): Ignoring split request from fadb2fe0ba32,33299,1732422708667, parent={ENCODED => a96be4dd9305efb471150cc3d77a7019, NAME => 'TestLogRolling-testLogRolling,,1732422709594.a96be4dd9305efb471150cc3d77a7019.', STARTKEY => '', ENDKEY => ''} because parent is unknown or not open
2024-11-24T04:32:03,873 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=a96be4dd9305efb471150cc3d77a7019, UNASSIGN}]
2024-11-24T04:32:03,874 INFO  [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=a96be4dd9305efb471150cc3d77a7019, UNASSIGN
2024-11-24T04:32:03,876 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=a96be4dd9305efb471150cc3d77a7019, regionState=CLOSING, regionLocation=fadb2fe0ba32,33299,1732422708667
2024-11-24T04:32:03,878 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=a96be4dd9305efb471150cc3d77a7019, UNASSIGN because future has completed
2024-11-24T04:32:03,879 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: true: evictOnSplit: true: evictOnClose: false
2024-11-24T04:32:03,879 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; CloseRegionProcedure a96be4dd9305efb471150cc3d77a7019, server=fadb2fe0ba32,33299,1732422708667}]
2024-11-24T04:32:04,036 INFO  [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(122): Close a96be4dd9305efb471150cc3d77a7019
2024-11-24T04:32:04,036 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(136): Unassign region: split region: true: evictCache: true
2024-11-24T04:32:04,037 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1722): Closing a96be4dd9305efb471150cc3d77a7019, disabling compactions & flushes
2024-11-24T04:32:04,037 INFO  [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1732422709594.a96be4dd9305efb471150cc3d77a7019.
2024-11-24T04:32:04,037 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1732422709594.a96be4dd9305efb471150cc3d77a7019.
2024-11-24T04:32:04,037 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1732422709594.a96be4dd9305efb471150cc3d77a7019. after waiting 0 ms
2024-11-24T04:32:04,037 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1732422709594.a96be4dd9305efb471150cc3d77a7019.
2024-11-24T04:32:04,037 INFO  [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(2902): Flushing a96be4dd9305efb471150cc3d77a7019 1/1 column families, dataSize=2.10 KB heapSize=2.50 KB
2024-11-24T04:32:04,041 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/.tmp/info/44a0bbc0bc374b0282c036160c35f7c3 is 1080, key is row0095/info:/1732422723845/Put/seqid=0
2024-11-24T04:32:04,046 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36545 is added to blk_1073741846_1022 (size=7112)
2024-11-24T04:32:04,046 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741846_1022 (size=7112)
2024-11-24T04:32:04,046 INFO  [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.10 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/.tmp/info/44a0bbc0bc374b0282c036160c35f7c3
2024-11-24T04:32:04,051 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/.tmp/info/44a0bbc0bc374b0282c036160c35f7c3 as hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/info/44a0bbc0bc374b0282c036160c35f7c3
2024-11-24T04:32:04,056 INFO  [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/info/44a0bbc0bc374b0282c036160c35f7c3, entries=2, sequenceid=123, filesize=6.9 K
2024-11-24T04:32:04,057 INFO  [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(3140): Finished flush of dataSize ~2.10 KB/2152, heapSize ~2.48 KB/2544, currentSize=0 B/0 for a96be4dd9305efb471150cc3d77a7019 in 20ms, sequenceid=123, compaction requested=true
2024-11-24T04:32:04,058 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732422709594.a96be4dd9305efb471150cc3d77a7019.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/info/57b6bc47fbde491780522b406a9e3c58, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/info/787d096725d842a2bf9c552c8218e92f, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/info/9f2c9ed2f9c0463481f67370f6b7cd0e, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/info/4656bb4ec4e140c482919c3942ecbedf, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/info/f3357cfa50ae4c8dabdb2037a0666cb1, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/info/da651c9fe3ae4ba894afc503b6b1343d] to archive
2024-11-24T04:32:04,059 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732422709594.a96be4dd9305efb471150cc3d77a7019.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-24T04:32:04,060 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732422709594.a96be4dd9305efb471150cc3d77a7019.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/info/57b6bc47fbde491780522b406a9e3c58 to hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/archive/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/info/57b6bc47fbde491780522b406a9e3c58
2024-11-24T04:32:04,062 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732422709594.a96be4dd9305efb471150cc3d77a7019.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/info/787d096725d842a2bf9c552c8218e92f to hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/archive/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/info/787d096725d842a2bf9c552c8218e92f
2024-11-24T04:32:04,063 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732422709594.a96be4dd9305efb471150cc3d77a7019.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/info/9f2c9ed2f9c0463481f67370f6b7cd0e to hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/archive/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/info/9f2c9ed2f9c0463481f67370f6b7cd0e
2024-11-24T04:32:04,064 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732422709594.a96be4dd9305efb471150cc3d77a7019.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/info/4656bb4ec4e140c482919c3942ecbedf to hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/archive/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/info/4656bb4ec4e140c482919c3942ecbedf
2024-11-24T04:32:04,065 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732422709594.a96be4dd9305efb471150cc3d77a7019.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/info/f3357cfa50ae4c8dabdb2037a0666cb1 to hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/archive/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/info/f3357cfa50ae4c8dabdb2037a0666cb1
2024-11-24T04:32:04,066 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732422709594.a96be4dd9305efb471150cc3d77a7019.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/info/da651c9fe3ae4ba894afc503b6b1343d to hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/archive/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/info/da651c9fe3ae4ba894afc503b6b1343d
2024-11-24T04:32:04,072 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/recovered.edits/126.seqid, newMaxSeqId=126, maxSeqId=1
2024-11-24T04:32:04,072 INFO  [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1732422709594.a96be4dd9305efb471150cc3d77a7019.
2024-11-24T04:32:04,072 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1676): Region close journal for a96be4dd9305efb471150cc3d77a7019:
Waiting for close lock at 1732422724037Running coprocessor pre-close hooks at 1732422724037Disabling compacts and flushes for region at 1732422724037Disabling writes for close at 1732422724037Obtaining lock to block concurrent updates at 1732422724037Preparing flush snapshotting stores in a96be4dd9305efb471150cc3d77a7019 at 1732422724037Finished memstore snapshotting TestLogRolling-testLogRolling,,1732422709594.a96be4dd9305efb471150cc3d77a7019., syncing WAL and waiting on mvcc, flushsize=dataSize=2152, getHeapSize=2544, getOffHeapSize=0, getCellsCount=2 at 1732422724037Flushing stores of TestLogRolling-testLogRolling,,1732422709594.a96be4dd9305efb471150cc3d77a7019. at 1732422724038 (+1 ms)Flushing a96be4dd9305efb471150cc3d77a7019/info: creating writer at 1732422724038Flushing a96be4dd9305efb471150cc3d77a7019/info: appending metadata at 1732422724041 (+3 ms)Flushing a96be4dd9305efb471150cc3d77a7019/info: closing flushed file at 1732422724041Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5a0124b4: reopening flushed file at 1732422724051 (+10 ms)Finished flush of dataSize ~2.10 KB/2152, heapSize ~2.48 KB/2544, currentSize=0 B/0 for a96be4dd9305efb471150cc3d77a7019 in 20ms, sequenceid=123, compaction requested=true at 1732422724057 (+6 ms)Writing region close event to WAL at 1732422724069 (+12 ms)Running coprocessor post-close hooks at 1732422724072 (+3 ms)Closed at 1732422724072
2024-11-24T04:32:04,074 INFO  [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(157): Closed a96be4dd9305efb471150cc3d77a7019
2024-11-24T04:32:04,075 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=a96be4dd9305efb471150cc3d77a7019, regionState=CLOSED
2024-11-24T04:32:04,077 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; CloseRegionProcedure a96be4dd9305efb471150cc3d77a7019, server=fadb2fe0ba32,33299,1732422708667 because future has completed
2024-11-24T04:32:04,080 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8
2024-11-24T04:32:04,080 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; CloseRegionProcedure a96be4dd9305efb471150cc3d77a7019, server=fadb2fe0ba32,33299,1732422708667 in 199 msec
2024-11-24T04:32:04,082 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7
2024-11-24T04:32:04,082 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=a96be4dd9305efb471150cc3d77a7019, UNASSIGN in 207 msec
2024-11-24T04:32:04,090 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:32:04,093 INFO  [PEWorker-4 {}] assignment.SplitTableRegionProcedure(728): pid=7 splitting 4 storefiles, region=a96be4dd9305efb471150cc3d77a7019, threads=4
2024-11-24T04:32:04,095 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/info/005ca3e2be6042eab9bd6f6c772b9469 for region: a96be4dd9305efb471150cc3d77a7019
2024-11-24T04:32:04,095 DEBUG [StoreFileSplitter-pool-2 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/info/6df4256bb0f1422cac4b53f1647f7eaa for region: a96be4dd9305efb471150cc3d77a7019
2024-11-24T04:32:04,095 DEBUG [StoreFileSplitter-pool-3 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/info/dda9939d8dc9423abdc85e4d4095a78f for region: a96be4dd9305efb471150cc3d77a7019
2024-11-24T04:32:04,095 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/info/44a0bbc0bc374b0282c036160c35f7c3 for region: a96be4dd9305efb471150cc3d77a7019
2024-11-24T04:32:04,110 DEBUG [StoreFileSplitter-pool-2 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/info/6df4256bb0f1422cac4b53f1647f7eaa, top=true
2024-11-24T04:32:04,110 DEBUG [StoreFileSplitter-pool-3 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/info/dda9939d8dc9423abdc85e4d4095a78f, top=true
2024-11-24T04:32:04,123 DEBUG [StoreFileSplitter-pool-1 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/info/44a0bbc0bc374b0282c036160c35f7c3, top=true
2024-11-24T04:32:04,130 INFO  [StoreFileSplitter-pool-3 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/TestLogRolling-testLogRolling=a96be4dd9305efb471150cc3d77a7019-dda9939d8dc9423abdc85e4d4095a78f for child: 3f22261b1394925c40b35fb57958000b, parent: a96be4dd9305efb471150cc3d77a7019
2024-11-24T04:32:04,130 INFO  [StoreFileSplitter-pool-2 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/TestLogRolling-testLogRolling=a96be4dd9305efb471150cc3d77a7019-6df4256bb0f1422cac4b53f1647f7eaa for child: 3f22261b1394925c40b35fb57958000b, parent: a96be4dd9305efb471150cc3d77a7019
2024-11-24T04:32:04,130 DEBUG [StoreFileSplitter-pool-2 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/info/6df4256bb0f1422cac4b53f1647f7eaa for region: a96be4dd9305efb471150cc3d77a7019
2024-11-24T04:32:04,130 DEBUG [StoreFileSplitter-pool-3 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/info/dda9939d8dc9423abdc85e4d4095a78f for region: a96be4dd9305efb471150cc3d77a7019
2024-11-24T04:32:04,131 INFO  [StoreFileSplitter-pool-1 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/TestLogRolling-testLogRolling=a96be4dd9305efb471150cc3d77a7019-44a0bbc0bc374b0282c036160c35f7c3 for child: 3f22261b1394925c40b35fb57958000b, parent: a96be4dd9305efb471150cc3d77a7019
2024-11-24T04:32:04,131 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/info/44a0bbc0bc374b0282c036160c35f7c3 for region: a96be4dd9305efb471150cc3d77a7019
2024-11-24T04:32:04,131 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36545 is added to blk_1073741847_1023 (size=27)
2024-11-24T04:32:04,132 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741847_1023 (size=27)
2024-11-24T04:32:04,141 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741848_1024 (size=27)
2024-11-24T04:32:04,141 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36545 is added to blk_1073741848_1024 (size=27)
2024-11-24T04:32:04,142 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/info/005ca3e2be6042eab9bd6f6c772b9469 for region: a96be4dd9305efb471150cc3d77a7019
2024-11-24T04:32:04,144 DEBUG [PEWorker-4 {}] assignment.SplitTableRegionProcedure(802): pid=7 split storefiles for region a96be4dd9305efb471150cc3d77a7019 Daughter A: [hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/9449b123c00b1a221a275bb41283b414/info/005ca3e2be6042eab9bd6f6c772b9469.a96be4dd9305efb471150cc3d77a7019] storefiles, Daughter B: [hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/005ca3e2be6042eab9bd6f6c772b9469.a96be4dd9305efb471150cc3d77a7019, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/TestLogRolling-testLogRolling=a96be4dd9305efb471150cc3d77a7019-44a0bbc0bc374b0282c036160c35f7c3, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/TestLogRolling-testLogRolling=a96be4dd9305efb471150cc3d77a7019-6df4256bb0f1422cac4b53f1647f7eaa, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/TestLogRolling-testLogRolling=a96be4dd9305efb471150cc3d77a7019-dda9939d8dc9423abdc85e4d4095a78f] storefiles.
2024-11-24T04:32:04,151 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741849_1025 (size=71)
2024-11-24T04:32:04,151 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36545 is added to blk_1073741849_1025 (size=71)
2024-11-24T04:32:04,153 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:32:04,164 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36545 is added to blk_1073741850_1026 (size=71)
2024-11-24T04:32:04,164 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741850_1026 (size=71)
2024-11-24T04:32:04,165 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:32:04,175 DEBUG [PEWorker-4 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/9449b123c00b1a221a275bb41283b414/recovered.edits/126.seqid, newMaxSeqId=126, maxSeqId=-1
2024-11-24T04:32:04,177 DEBUG [PEWorker-4 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/recovered.edits/126.seqid, newMaxSeqId=126, maxSeqId=-1
2024-11-24T04:32:04,179 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1732422709594.a96be4dd9305efb471150cc3d77a7019.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1732422724178"},{"qualifier":"splitA","vlen":70,"tag":[],"timestamp":"1732422724178"},{"qualifier":"splitB","vlen":70,"tag":[],"timestamp":"1732422724178"}]},"ts":"1732422724178"}
2024-11-24T04:32:04,179 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1732422723860.9449b123c00b1a221a275bb41283b414.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1732422724178"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732422724178"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1732422724178"}]},"ts":"1732422724178"}
2024-11-24T04:32:04,179 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,row0062,1732422723860.3f22261b1394925c40b35fb57958000b.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1732422724178"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732422724178"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1732422724178"}]},"ts":"1732422724178"}
2024-11-24T04:32:04,196 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=9449b123c00b1a221a275bb41283b414, ASSIGN}, {pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=3f22261b1394925c40b35fb57958000b, ASSIGN}]
2024-11-24T04:32:04,197 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=9449b123c00b1a221a275bb41283b414, ASSIGN
2024-11-24T04:32:04,197 INFO  [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=3f22261b1394925c40b35fb57958000b, ASSIGN
2024-11-24T04:32:04,198 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=9449b123c00b1a221a275bb41283b414, ASSIGN; state=SPLITTING_NEW, location=fadb2fe0ba32,33299,1732422708667; forceNewPlan=false, retain=false
2024-11-24T04:32:04,198 INFO  [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=3f22261b1394925c40b35fb57958000b, ASSIGN; state=SPLITTING_NEW, location=fadb2fe0ba32,33299,1732422708667; forceNewPlan=false, retain=false
2024-11-24T04:32:04,349 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=9449b123c00b1a221a275bb41283b414, regionState=OPENING, regionLocation=fadb2fe0ba32,33299,1732422708667
2024-11-24T04:32:04,349 INFO  [PEWorker-2 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=3f22261b1394925c40b35fb57958000b, regionState=OPENING, regionLocation=fadb2fe0ba32,33299,1732422708667
2024-11-24T04:32:04,352 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=3f22261b1394925c40b35fb57958000b, ASSIGN because future has completed
2024-11-24T04:32:04,352 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3f22261b1394925c40b35fb57958000b, server=fadb2fe0ba32,33299,1732422708667}]
2024-11-24T04:32:04,352 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=9449b123c00b1a221a275bb41283b414, ASSIGN because future has completed
2024-11-24T04:32:04,353 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=13, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9449b123c00b1a221a275bb41283b414, server=fadb2fe0ba32,33299,1732422708667}]
2024-11-24T04:32:04,508 INFO  [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,row0062,1732422723860.3f22261b1394925c40b35fb57958000b.
2024-11-24T04:32:04,508 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 3f22261b1394925c40b35fb57958000b, NAME => 'TestLogRolling-testLogRolling,row0062,1732422723860.3f22261b1394925c40b35fb57958000b.', STARTKEY => 'row0062', ENDKEY => ''}
2024-11-24T04:32:04,508 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling 3f22261b1394925c40b35fb57958000b
2024-11-24T04:32:04,508 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,row0062,1732422723860.3f22261b1394925c40b35fb57958000b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T04:32:04,508 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 3f22261b1394925c40b35fb57958000b
2024-11-24T04:32:04,508 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 3f22261b1394925c40b35fb57958000b
2024-11-24T04:32:04,510 INFO  [StoreOpener-3f22261b1394925c40b35fb57958000b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 3f22261b1394925c40b35fb57958000b 
2024-11-24T04:32:04,510 INFO  [StoreOpener-3f22261b1394925c40b35fb57958000b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3f22261b1394925c40b35fb57958000b columnFamilyName info
2024-11-24T04:32:04,511 DEBUG [StoreOpener-3f22261b1394925c40b35fb57958000b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:32:04,520 DEBUG [StoreOpener-3f22261b1394925c40b35fb57958000b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/005ca3e2be6042eab9bd6f6c772b9469.a96be4dd9305efb471150cc3d77a7019->hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/info/005ca3e2be6042eab9bd6f6c772b9469-top
2024-11-24T04:32:04,525 DEBUG [StoreOpener-3f22261b1394925c40b35fb57958000b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/TestLogRolling-testLogRolling=a96be4dd9305efb471150cc3d77a7019-44a0bbc0bc374b0282c036160c35f7c3
2024-11-24T04:32:04,528 DEBUG [StoreOpener-3f22261b1394925c40b35fb57958000b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/TestLogRolling-testLogRolling=a96be4dd9305efb471150cc3d77a7019-6df4256bb0f1422cac4b53f1647f7eaa
2024-11-24T04:32:04,532 DEBUG [StoreOpener-3f22261b1394925c40b35fb57958000b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/TestLogRolling-testLogRolling=a96be4dd9305efb471150cc3d77a7019-dda9939d8dc9423abdc85e4d4095a78f
2024-11-24T04:32:04,532 INFO  [StoreOpener-3f22261b1394925c40b35fb57958000b-1 {}] regionserver.HStore(327): Store=3f22261b1394925c40b35fb57958000b/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-24T04:32:04,532 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 3f22261b1394925c40b35fb57958000b
2024-11-24T04:32:04,533 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b
2024-11-24T04:32:04,534 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b
2024-11-24T04:32:04,535 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 3f22261b1394925c40b35fb57958000b
2024-11-24T04:32:04,535 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 3f22261b1394925c40b35fb57958000b
2024-11-24T04:32:04,536 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 3f22261b1394925c40b35fb57958000b
2024-11-24T04:32:04,537 INFO  [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 3f22261b1394925c40b35fb57958000b; next sequenceid=127; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=763464, jitterRate=-0.02920648455619812}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-24T04:32:04,538 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3f22261b1394925c40b35fb57958000b
2024-11-24T04:32:04,538 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 3f22261b1394925c40b35fb57958000b:
Running coprocessor pre-open hook at 1732422724508Writing region info on filesystem at 1732422724508Initializing all the Stores at 1732422724509 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422724509Cleaning up temporary data from old regions at 1732422724535 (+26 ms)Running coprocessor post-open hooks at 1732422724538 (+3 ms)Region opened successfully at 1732422724538
2024-11-24T04:32:04,539 INFO  [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,row0062,1732422723860.3f22261b1394925c40b35fb57958000b., pid=12, masterSystemTime=1732422724504
2024-11-24T04:32:04,540 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.CompactSplit(403): Add compact mark for store 3f22261b1394925c40b35fb57958000b:info, priority=-2147483648, current under compaction store size is 1
2024-11-24T04:32:04,540 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking
2024-11-24T04:32:04,540 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-24T04:32:04,541 INFO  [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.HStore(1527): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,row0062,1732422723860.3f22261b1394925c40b35fb57958000b.
2024-11-24T04:32:04,541 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.HStore(1541): 3f22261b1394925c40b35fb57958000b/info is initiating minor compaction (all files)
2024-11-24T04:32:04,541 INFO  [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f22261b1394925c40b35fb57958000b/info in TestLogRolling-testLogRolling,row0062,1732422723860.3f22261b1394925c40b35fb57958000b.
2024-11-24T04:32:04,541 INFO  [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/005ca3e2be6042eab9bd6f6c772b9469.a96be4dd9305efb471150cc3d77a7019->hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/info/005ca3e2be6042eab9bd6f6c772b9469-top, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/TestLogRolling-testLogRolling=a96be4dd9305efb471150cc3d77a7019-6df4256bb0f1422cac4b53f1647f7eaa, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/TestLogRolling-testLogRolling=a96be4dd9305efb471150cc3d77a7019-dda9939d8dc9423abdc85e4d4095a78f, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/TestLogRolling-testLogRolling=a96be4dd9305efb471150cc3d77a7019-44a0bbc0bc374b0282c036160c35f7c3] into tmpdir=hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/.tmp, totalSize=120.8 K
2024-11-24T04:32:04,542 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,row0062,1732422723860.3f22261b1394925c40b35fb57958000b.
2024-11-24T04:32:04,542 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 005ca3e2be6042eab9bd6f6c772b9469.a96be4dd9305efb471150cc3d77a7019, keycount=32, bloomtype=ROW, size=73.6 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732422719680
2024-11-24T04:32:04,542 INFO  [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,row0062,1732422723860.3f22261b1394925c40b35fb57958000b.
2024-11-24T04:32:04,542 INFO  [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,,1732422723860.9449b123c00b1a221a275bb41283b414.
2024-11-24T04:32:04,542 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7752): Opening region: {ENCODED => 9449b123c00b1a221a275bb41283b414, NAME => 'TestLogRolling-testLogRolling,,1732422723860.9449b123c00b1a221a275bb41283b414.', STARTKEY => '', ENDKEY => 'row0062'}
2024-11-24T04:32:04,542 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] compactions.Compactor(225): Compacting TestLogRolling-testLogRolling=a96be4dd9305efb471150cc3d77a7019-6df4256bb0f1422cac4b53f1647f7eaa, keycount=14, bloomtype=ROW, size=19.6 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732422723792
2024-11-24T04:32:04,543 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling 9449b123c00b1a221a275bb41283b414
2024-11-24T04:32:04,543 INFO  [PEWorker-2 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=3f22261b1394925c40b35fb57958000b, regionState=OPEN, openSeqNum=127, regionLocation=fadb2fe0ba32,33299,1732422708667
2024-11-24T04:32:04,543 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] compactions.Compactor(225): Compacting TestLogRolling-testLogRolling=a96be4dd9305efb471150cc3d77a7019-dda9939d8dc9423abdc85e4d4095a78f, keycount=15, bloomtype=ROW, size=20.6 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732422723815
2024-11-24T04:32:04,543 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1732422723860.9449b123c00b1a221a275bb41283b414.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T04:32:04,543 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7794): checking encryption for 9449b123c00b1a221a275bb41283b414
2024-11-24T04:32:04,543 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7797): checking classloading for 9449b123c00b1a221a275bb41283b414
2024-11-24T04:32:04,543 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] compactions.Compactor(225): Compacting TestLogRolling-testLogRolling=a96be4dd9305efb471150cc3d77a7019-44a0bbc0bc374b0282c036160c35f7c3, keycount=2, bloomtype=ROW, size=6.9 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1732422723845
2024-11-24T04:32:04,544 INFO  [StoreOpener-9449b123c00b1a221a275bb41283b414-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 9449b123c00b1a221a275bb41283b414 
2024-11-24T04:32:04,545 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33299 {}] regionserver.HRegion(8855): Flush requested on 1588230740
2024-11-24T04:32:04,545 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all.
2024-11-24T04:32:04,545 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.15 KB heapSize=9 KB
2024-11-24T04:32:04,545 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3f22261b1394925c40b35fb57958000b, server=fadb2fe0ba32,33299,1732422708667 because future has completed
2024-11-24T04:32:04,545 INFO  [StoreOpener-9449b123c00b1a221a275bb41283b414-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9449b123c00b1a221a275bb41283b414 columnFamilyName info
2024-11-24T04:32:04,545 DEBUG [StoreOpener-9449b123c00b1a221a275bb41283b414-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:32:04,550 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11
2024-11-24T04:32:04,550 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 3f22261b1394925c40b35fb57958000b, server=fadb2fe0ba32,33299,1732422708667 in 194 msec
2024-11-24T04:32:04,552 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=3f22261b1394925c40b35fb57958000b, ASSIGN in 354 msec
2024-11-24T04:32:04,554 DEBUG [StoreOpener-9449b123c00b1a221a275bb41283b414-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/9449b123c00b1a221a275bb41283b414/info/005ca3e2be6042eab9bd6f6c772b9469.a96be4dd9305efb471150cc3d77a7019->hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/info/005ca3e2be6042eab9bd6f6c772b9469-bottom
2024-11-24T04:32:04,555 INFO  [StoreOpener-9449b123c00b1a221a275bb41283b414-1 {}] regionserver.HStore(327): Store=9449b123c00b1a221a275bb41283b414/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-24T04:32:04,555 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1038): replaying wal for 9449b123c00b1a221a275bb41283b414
2024-11-24T04:32:04,556 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/9449b123c00b1a221a275bb41283b414
2024-11-24T04:32:04,557 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/9449b123c00b1a221a275bb41283b414
2024-11-24T04:32:04,557 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1048): stopping wal replay for 9449b123c00b1a221a275bb41283b414
2024-11-24T04:32:04,557 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1060): Cleaning up temporary data for 9449b123c00b1a221a275bb41283b414
2024-11-24T04:32:04,558 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1093): writing seq id for 9449b123c00b1a221a275bb41283b414
2024-11-24T04:32:04,559 INFO  [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1114): Opened 9449b123c00b1a221a275bb41283b414; next sequenceid=127; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=724240, jitterRate=-0.07908150553703308}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-24T04:32:04,559 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9449b123c00b1a221a275bb41283b414
2024-11-24T04:32:04,559 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1006): Region open journal for 9449b123c00b1a221a275bb41283b414:
Running coprocessor pre-open hook at 1732422724543Writing region info on filesystem at 1732422724543Initializing all the Stores at 1732422724544 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422724544Cleaning up temporary data from old regions at 1732422724557 (+13 ms)Running coprocessor post-open hooks at 1732422724559 (+2 ms)Region opened successfully at 1732422724559
2024-11-24T04:32:04,560 INFO  [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,,1732422723860.9449b123c00b1a221a275bb41283b414., pid=13, masterSystemTime=1732422724504
2024-11-24T04:32:04,561 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.CompactSplit(403): Add compact mark for store 9449b123c00b1a221a275bb41283b414:info, priority=-2147483648, current under compaction store size is 2
2024-11-24T04:32:04,561 DEBUG [RS:0;fadb2fe0ba32:33299-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking
2024-11-24T04:32:04,561 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-24T04:32:04,561 INFO  [RS:0;fadb2fe0ba32:33299-longCompactions-0 {}] regionserver.HStore(1527): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,,1732422723860.9449b123c00b1a221a275bb41283b414.
2024-11-24T04:32:04,561 DEBUG [RS:0;fadb2fe0ba32:33299-longCompactions-0 {}] regionserver.HStore(1541): 9449b123c00b1a221a275bb41283b414/info is initiating minor compaction (all files)
2024-11-24T04:32:04,561 INFO  [RS:0;fadb2fe0ba32:33299-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9449b123c00b1a221a275bb41283b414/info in TestLogRolling-testLogRolling,,1732422723860.9449b123c00b1a221a275bb41283b414.
2024-11-24T04:32:04,561 INFO  [RS:0;fadb2fe0ba32:33299-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/9449b123c00b1a221a275bb41283b414/info/005ca3e2be6042eab9bd6f6c772b9469.a96be4dd9305efb471150cc3d77a7019->hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/info/005ca3e2be6042eab9bd6f6c772b9469-bottom] into tmpdir=hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/9449b123c00b1a221a275bb41283b414/.tmp, totalSize=73.6 K
2024-11-24T04:32:04,562 DEBUG [RS:0;fadb2fe0ba32:33299-longCompactions-0 {}] compactions.Compactor(225): Compacting 005ca3e2be6042eab9bd6f6c772b9469.a96be4dd9305efb471150cc3d77a7019, keycount=32, bloomtype=ROW, size=73.6 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732422719680
2024-11-24T04:32:04,563 DEBUG [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,,1732422723860.9449b123c00b1a221a275bb41283b414.
2024-11-24T04:32:04,563 INFO  [RS_OPEN_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,,1732422723860.9449b123c00b1a221a275bb41283b414.
2024-11-24T04:32:04,564 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=9449b123c00b1a221a275bb41283b414, regionState=OPEN, openSeqNum=127, regionLocation=fadb2fe0ba32,33299,1732422708667
2024-11-24T04:32:04,564 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/hbase/meta/1588230740/.tmp/info/9588e7f22db94cc4acb06707ade34a1f is 193, key is TestLogRolling-testLogRolling,row0062,1732422723860.3f22261b1394925c40b35fb57958000b./info:regioninfo/1732422724543/Put/seqid=0
2024-11-24T04:32:04,566 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9449b123c00b1a221a275bb41283b414, server=fadb2fe0ba32,33299,1732422708667 because future has completed
2024-11-24T04:32:04,572 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=13, resume processing ppid=10
2024-11-24T04:32:04,572 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 9449b123c00b1a221a275bb41283b414, server=fadb2fe0ba32,33299,1732422708667 in 217 msec
2024-11-24T04:32:04,574 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=7
2024-11-24T04:32:04,574 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=9449b123c00b1a221a275bb41283b414, ASSIGN in 376 msec
2024-11-24T04:32:04,576 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36545 is added to blk_1073741851_1027 (size=9882)
2024-11-24T04:32:04,576 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741851_1027 (size=9882)
2024-11-24T04:32:04,576 INFO  [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f22261b1394925c40b35fb57958000b#info#compaction#66 average throughput is 17.96 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-24T04:32:04,577 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/.tmp/info/55e010b042dc44d0a08b905cda87de25 is 1080, key is row0062/info:/1732422721783/Put/seqid=0
2024-11-24T04:32:04,577 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.95 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/hbase/meta/1588230740/.tmp/info/9588e7f22db94cc4acb06707ade34a1f
2024-11-24T04:32:04,577 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=a96be4dd9305efb471150cc3d77a7019, daughterA=9449b123c00b1a221a275bb41283b414, daughterB=3f22261b1394925c40b35fb57958000b in 714 msec
2024-11-24T04:32:04,582 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741852_1028 (size=43081)
2024-11-24T04:32:04,582 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36545 is added to blk_1073741852_1028 (size=43081)
2024-11-24T04:32:04,582 INFO  [RS:0;fadb2fe0ba32:33299-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9449b123c00b1a221a275bb41283b414#info#compaction#67 average throughput is 31.30 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-24T04:32:04,583 DEBUG [RS:0;fadb2fe0ba32:33299-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/9449b123c00b1a221a275bb41283b414/.tmp/info/2d4ecb9595d548a4a93f74c92a959842 is 1080, key is row0001/info:/1732422719680/Put/seqid=0
2024-11-24T04:32:04,586 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741853_1029 (size=70862)
2024-11-24T04:32:04,587 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36545 is added to blk_1073741853_1029 (size=70862)
2024-11-24T04:32:04,589 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/.tmp/info/55e010b042dc44d0a08b905cda87de25 as hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/55e010b042dc44d0a08b905cda87de25
2024-11-24T04:32:04,593 DEBUG [RS:0;fadb2fe0ba32:33299-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/9449b123c00b1a221a275bb41283b414/.tmp/info/2d4ecb9595d548a4a93f74c92a959842 as hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/9449b123c00b1a221a275bb41283b414/info/2d4ecb9595d548a4a93f74c92a959842
2024-11-24T04:32:04,594 INFO  [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3f22261b1394925c40b35fb57958000b/info of 3f22261b1394925c40b35fb57958000b into 55e010b042dc44d0a08b905cda87de25(size=42.1 K), total size for store is 42.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-24T04:32:04,594 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f22261b1394925c40b35fb57958000b:

2024-11-24T04:32:04,595 INFO  [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732422723860.3f22261b1394925c40b35fb57958000b., storeName=3f22261b1394925c40b35fb57958000b/info, priority=12, startTime=1732422724540; duration=0sec
2024-11-24T04:32:04,595 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-24T04:32:04,595 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f22261b1394925c40b35fb57958000b:info
2024-11-24T04:32:04,598 INFO  [RS:0;fadb2fe0ba32:33299-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 1 (all) file(s) in 9449b123c00b1a221a275bb41283b414/info of 9449b123c00b1a221a275bb41283b414 into 2d4ecb9595d548a4a93f74c92a959842(size=69.2 K), total size for store is 69.2 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-24T04:32:04,598 DEBUG [RS:0;fadb2fe0ba32:33299-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9449b123c00b1a221a275bb41283b414:

2024-11-24T04:32:04,598 INFO  [RS:0;fadb2fe0ba32:33299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1732422723860.9449b123c00b1a221a275bb41283b414., storeName=9449b123c00b1a221a275bb41283b414/info, priority=15, startTime=1732422724560; duration=0sec
2024-11-24T04:32:04,598 DEBUG [RS:0;fadb2fe0ba32:33299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-24T04:32:04,598 DEBUG [RS:0;fadb2fe0ba32:33299-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9449b123c00b1a221a275bb41283b414:info
2024-11-24T04:32:04,600 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/hbase/meta/1588230740/.tmp/ns/7a3a984e7932430a8e97b673b5df38f6 is 43, key is default/ns:d/1732422709476/Put/seqid=0
2024-11-24T04:32:04,604 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741854_1030 (size=5153)
2024-11-24T04:32:04,605 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36545 is added to blk_1073741854_1030 (size=5153)
2024-11-24T04:32:04,605 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/hbase/meta/1588230740/.tmp/ns/7a3a984e7932430a8e97b673b5df38f6
2024-11-24T04:32:04,623 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/hbase/meta/1588230740/.tmp/table/7dbf40c66002440d9f398474d6df8fdd is 65, key is TestLogRolling-testLogRolling/table:state/1732422709952/Put/seqid=0
2024-11-24T04:32:04,628 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741855_1031 (size=5340)
2024-11-24T04:32:04,628 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36545 is added to blk_1073741855_1031 (size=5340)
2024-11-24T04:32:04,628 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=122 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/hbase/meta/1588230740/.tmp/table/7dbf40c66002440d9f398474d6df8fdd
2024-11-24T04:32:04,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/hbase/meta/1588230740/.tmp/info/9588e7f22db94cc4acb06707ade34a1f as hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/hbase/meta/1588230740/info/9588e7f22db94cc4acb06707ade34a1f
2024-11-24T04:32:04,638 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/hbase/meta/1588230740/info/9588e7f22db94cc4acb06707ade34a1f, entries=30, sequenceid=17, filesize=9.7 K
2024-11-24T04:32:04,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/hbase/meta/1588230740/.tmp/ns/7a3a984e7932430a8e97b673b5df38f6 as hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/hbase/meta/1588230740/ns/7a3a984e7932430a8e97b673b5df38f6
2024-11-24T04:32:04,643 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/hbase/meta/1588230740/ns/7a3a984e7932430a8e97b673b5df38f6, entries=2, sequenceid=17, filesize=5.0 K
2024-11-24T04:32:04,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/hbase/meta/1588230740/.tmp/table/7dbf40c66002440d9f398474d6df8fdd as hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/hbase/meta/1588230740/table/7dbf40c66002440d9f398474d6df8fdd
2024-11-24T04:32:04,649 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/hbase/meta/1588230740/table/7dbf40c66002440d9f398474d6df8fdd, entries=2, sequenceid=17, filesize=5.2 K
2024-11-24T04:32:04,650 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~5.15 KB/5269, heapSize ~8.70 KB/8912, currentSize=670 B/670 for 1588230740 in 105ms, sequenceid=17, compaction requested=false
2024-11-24T04:32:04,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1588230740:

2024-11-24T04:32:04,850 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:04,850 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:05,850 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:05,850 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:05,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33299 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:46948 deadline: 1732422735848, exception=org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1732422709594.a96be4dd9305efb471150cc3d77a7019. is not online on fadb2fe0ba32,33299,1732422708667
2024-11-24T04:32:05,874 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestLogRolling-testLogRolling,,1732422709594.a96be4dd9305efb471150cc3d77a7019., hostname=fadb2fe0ba32,33299,1732422708667, seqNum=2 , the old value is region=TestLogRolling-testLogRolling,,1732422709594.a96be4dd9305efb471150cc3d77a7019., hostname=fadb2fe0ba32,33299,1732422708667, seqNum=2, error=org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1732422709594.a96be4dd9305efb471150cc3d77a7019. is not online on fadb2fe0ba32,33299,1732422708667
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186)
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2943)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-11-24T04:32:05,875 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestLogRolling-testLogRolling,,1732422709594.a96be4dd9305efb471150cc3d77a7019., hostname=fadb2fe0ba32,33299,1732422708667, seqNum=2 is org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1732422709594.a96be4dd9305efb471150cc3d77a7019. is not online on fadb2fe0ba32,33299,1732422708667
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186)
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2943)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-11-24T04:32:05,875 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=TestLogRolling-testLogRolling,,1732422709594.a96be4dd9305efb471150cc3d77a7019., hostname=fadb2fe0ba32,33299,1732422708667, seqNum=2 from cache
2024-11-24T04:32:06,851 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:06,851 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:07,851 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:07,851 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:08,852 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:08,852 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:09,574 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-24T04:32:09,576 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:32:09,576 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:32:09,576 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:32:09,576 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:32:09,576 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:32:09,576 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:32:09,577 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:32:09,577 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:32:09,595 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:32:09,595 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:32:09,595 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:32:09,595 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:32:09,596 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:32:09,596 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:32:09,599 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:32:09,599 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:32:09,599 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:32:09,602 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-24T04:32:09,853 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:09,853 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:10,853 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:10,853 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:11,854 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:11,854 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:12,854 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:12,854 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:13,855 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:13,855 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:14,856 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:14,856 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:15,856 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:15,856 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:15,928 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRolling', row='row0097', locateType=CURRENT is [region=TestLogRolling-testLogRolling,row0062,1732422723860.3f22261b1394925c40b35fb57958000b., hostname=fadb2fe0ba32,33299,1732422708667, seqNum=127]
2024-11-24T04:32:15,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33299 {}] regionserver.HRegion(8855): Flush requested on 3f22261b1394925c40b35fb57958000b
2024-11-24T04:32:15,939 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3f22261b1394925c40b35fb57958000b 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-24T04:32:15,944 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/.tmp/info/71241d62e1264a959e1b61319644b704 is 1080, key is row0097/info:/1732422735929/Put/seqid=0
2024-11-24T04:32:15,954 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36545 is added to blk_1073741856_1032 (size=12516)
2024-11-24T04:32:15,954 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741856_1032 (size=12516)
2024-11-24T04:32:15,955 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/.tmp/info/71241d62e1264a959e1b61319644b704
2024-11-24T04:32:15,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/.tmp/info/71241d62e1264a959e1b61319644b704 as hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/71241d62e1264a959e1b61319644b704
2024-11-24T04:32:15,965 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/71241d62e1264a959e1b61319644b704, entries=7, sequenceid=137, filesize=12.2 K
2024-11-24T04:32:15,966 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=17.86 KB/18292 for 3f22261b1394925c40b35fb57958000b in 27ms, sequenceid=137, compaction requested=false
2024-11-24T04:32:15,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3f22261b1394925c40b35fb57958000b:

2024-11-24T04:32:15,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33299 {}] regionserver.HRegion(8855): Flush requested on 3f22261b1394925c40b35fb57958000b
2024-11-24T04:32:15,968 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3f22261b1394925c40b35fb57958000b 1/1 column families, dataSize=19.96 KB heapSize=21.63 KB
2024-11-24T04:32:15,972 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/.tmp/info/e951e9e1b9214854a0f6023fb3afc0e8 is 1080, key is row0104/info:/1732422735940/Put/seqid=0
2024-11-24T04:32:15,979 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36545 is added to blk_1073741857_1033 (size=25472)
2024-11-24T04:32:15,979 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741857_1033 (size=25472)
2024-11-24T04:32:15,979 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=19.96 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/.tmp/info/e951e9e1b9214854a0f6023fb3afc0e8
2024-11-24T04:32:15,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/.tmp/info/e951e9e1b9214854a0f6023fb3afc0e8 as hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/e951e9e1b9214854a0f6023fb3afc0e8
2024-11-24T04:32:15,988 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/e951e9e1b9214854a0f6023fb3afc0e8, entries=19, sequenceid=159, filesize=24.9 K
2024-11-24T04:32:15,989 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~19.96 KB/20444, heapSize ~21.61 KB/22128, currentSize=6.30 KB/6456 for 3f22261b1394925c40b35fb57958000b in 21ms, sequenceid=159, compaction requested=true
2024-11-24T04:32:15,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3f22261b1394925c40b35fb57958000b:

2024-11-24T04:32:15,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f22261b1394925c40b35fb57958000b:info, priority=-2147483648, current under compaction store size is 1
2024-11-24T04:32:15,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-24T04:32:15,989 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-24T04:32:15,990 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 81069 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-24T04:32:15,990 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.HStore(1541): 3f22261b1394925c40b35fb57958000b/info is initiating minor compaction (all files)
2024-11-24T04:32:15,990 INFO  [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f22261b1394925c40b35fb57958000b/info in TestLogRolling-testLogRolling,row0062,1732422723860.3f22261b1394925c40b35fb57958000b.
2024-11-24T04:32:15,990 INFO  [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/55e010b042dc44d0a08b905cda87de25, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/71241d62e1264a959e1b61319644b704, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/e951e9e1b9214854a0f6023fb3afc0e8] into tmpdir=hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/.tmp, totalSize=79.2 K
2024-11-24T04:32:15,991 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55e010b042dc44d0a08b905cda87de25, keycount=35, bloomtype=ROW, size=42.1 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1732422721783
2024-11-24T04:32:15,991 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71241d62e1264a959e1b61319644b704, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732422735929
2024-11-24T04:32:15,992 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] compactions.Compactor(225): Compacting e951e9e1b9214854a0f6023fb3afc0e8, keycount=19, bloomtype=ROW, size=24.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732422735940
2024-11-24T04:32:16,001 INFO  [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f22261b1394925c40b35fb57958000b#info#compaction#72 average throughput is 31.30 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-24T04:32:16,001 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/.tmp/info/d062343cbbd34e2c992e85c468d9976f is 1080, key is row0062/info:/1732422721783/Put/seqid=0
2024-11-24T04:32:16,005 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36545 is added to blk_1073741858_1034 (size=71283)
2024-11-24T04:32:16,005 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741858_1034 (size=71283)
2024-11-24T04:32:16,010 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/.tmp/info/d062343cbbd34e2c992e85c468d9976f as hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/d062343cbbd34e2c992e85c468d9976f
2024-11-24T04:32:16,016 INFO  [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f22261b1394925c40b35fb57958000b/info of 3f22261b1394925c40b35fb57958000b into d062343cbbd34e2c992e85c468d9976f(size=69.6 K), total size for store is 69.6 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-24T04:32:16,016 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f22261b1394925c40b35fb57958000b:

2024-11-24T04:32:16,016 INFO  [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732422723860.3f22261b1394925c40b35fb57958000b., storeName=3f22261b1394925c40b35fb57958000b/info, priority=13, startTime=1732422735989; duration=0sec
2024-11-24T04:32:16,016 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-24T04:32:16,016 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f22261b1394925c40b35fb57958000b:info
2024-11-24T04:32:16,857 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:16,857 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:17,858 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:17,858 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:17,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33299 {}] regionserver.HRegion(8855): Flush requested on 3f22261b1394925c40b35fb57958000b
2024-11-24T04:32:17,980 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3f22261b1394925c40b35fb57958000b 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-24T04:32:17,984 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/.tmp/info/b1c4842d369d499ba01af0994f04b0b0 is 1080, key is row0123/info:/1732422735969/Put/seqid=0
2024-11-24T04:32:17,988 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36545 is added to blk_1073741859_1035 (size=12516)
2024-11-24T04:32:17,989 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741859_1035 (size=12516)
2024-11-24T04:32:17,989 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/.tmp/info/b1c4842d369d499ba01af0994f04b0b0
2024-11-24T04:32:17,995 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/.tmp/info/b1c4842d369d499ba01af0994f04b0b0 as hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/b1c4842d369d499ba01af0994f04b0b0
2024-11-24T04:32:18,000 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/b1c4842d369d499ba01af0994f04b0b0, entries=7, sequenceid=170, filesize=12.2 K
2024-11-24T04:32:18,000 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=14.71 KB/15064 for 3f22261b1394925c40b35fb57958000b in 21ms, sequenceid=170, compaction requested=false
2024-11-24T04:32:18,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3f22261b1394925c40b35fb57958000b:

2024-11-24T04:32:18,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33299 {}] regionserver.HRegion(8855): Flush requested on 3f22261b1394925c40b35fb57958000b
2024-11-24T04:32:18,002 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3f22261b1394925c40b35fb57958000b 1/1 column families, dataSize=15.76 KB heapSize=17.13 KB
2024-11-24T04:32:18,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/.tmp/info/9fd4851a42f641f1b6f934635d604f43 is 1080, key is row0130/info:/1732422737981/Put/seqid=0
2024-11-24T04:32:18,014 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741860_1036 (size=21156)
2024-11-24T04:32:18,014 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36545 is added to blk_1073741860_1036 (size=21156)
2024-11-24T04:32:18,014 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.76 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/.tmp/info/9fd4851a42f641f1b6f934635d604f43
2024-11-24T04:32:18,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/.tmp/info/9fd4851a42f641f1b6f934635d604f43 as hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/9fd4851a42f641f1b6f934635d604f43
2024-11-24T04:32:18,025 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/9fd4851a42f641f1b6f934635d604f43, entries=15, sequenceid=188, filesize=20.7 K
2024-11-24T04:32:18,025 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit.
org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=3f22261b1394925c40b35fb57958000b, server=fadb2fe0ba32,33299,1732422708667
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT]
2024-11-24T04:32:18,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33299 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:46948 deadline: 1732422748024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=3f22261b1394925c40b35fb57958000b, server=fadb2fe0ba32,33299,1732422708667
2024-11-24T04:32:18,026 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestLogRolling-testLogRolling,row0062,1732422723860.3f22261b1394925c40b35fb57958000b., hostname=fadb2fe0ba32,33299,1732422708667, seqNum=127 , the old value is region=TestLogRolling-testLogRolling,row0062,1732422723860.3f22261b1394925c40b35fb57958000b., hostname=fadb2fe0ba32,33299,1732422708667, seqNum=127, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=3f22261b1394925c40b35fb57958000b, server=fadb2fe0ba32,33299,1732422708667
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308)
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329)
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216)
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-11-24T04:32:18,026 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestLogRolling-testLogRolling,row0062,1732422723860.3f22261b1394925c40b35fb57958000b., hostname=fadb2fe0ba32,33299,1732422708667, seqNum=127 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=3f22261b1394925c40b35fb57958000b, server=fadb2fe0ba32,33299,1732422708667
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308)
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329)
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216)
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-11-24T04:32:18,026 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestLogRolling-testLogRolling,row0062,1732422723860.3f22261b1394925c40b35fb57958000b., hostname=fadb2fe0ba32,33299,1732422708667, seqNum=127 because the exception is null or not the one we care about
2024-11-24T04:32:18,026 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~15.76 KB/16140, heapSize ~17.11 KB/17520, currentSize=14.71 KB/15064 for 3f22261b1394925c40b35fb57958000b in 24ms, sequenceid=188, compaction requested=true
2024-11-24T04:32:18,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3f22261b1394925c40b35fb57958000b:

2024-11-24T04:32:18,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f22261b1394925c40b35fb57958000b:info, priority=-2147483648, current under compaction store size is 1
2024-11-24T04:32:18,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-24T04:32:18,026 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-24T04:32:18,027 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 104955 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-24T04:32:18,027 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.HStore(1541): 3f22261b1394925c40b35fb57958000b/info is initiating minor compaction (all files)
2024-11-24T04:32:18,027 INFO  [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f22261b1394925c40b35fb57958000b/info in TestLogRolling-testLogRolling,row0062,1732422723860.3f22261b1394925c40b35fb57958000b.
2024-11-24T04:32:18,027 INFO  [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/d062343cbbd34e2c992e85c468d9976f, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/b1c4842d369d499ba01af0994f04b0b0, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/9fd4851a42f641f1b6f934635d604f43] into tmpdir=hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/.tmp, totalSize=102.5 K
2024-11-24T04:32:18,027 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] compactions.Compactor(225): Compacting d062343cbbd34e2c992e85c468d9976f, keycount=61, bloomtype=ROW, size=69.6 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732422721783
2024-11-24T04:32:18,028 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1c4842d369d499ba01af0994f04b0b0, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732422735969
2024-11-24T04:32:18,028 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9fd4851a42f641f1b6f934635d604f43, keycount=15, bloomtype=ROW, size=20.7 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732422737981
2024-11-24T04:32:18,038 INFO  [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f22261b1394925c40b35fb57958000b#info#compaction#75 average throughput is 85.17 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-24T04:32:18,038 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/.tmp/info/21e4e452e8d34499ad99a24d2b60a75d is 1080, key is row0062/info:/1732422721783/Put/seqid=0
2024-11-24T04:32:18,041 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741861_1037 (size=95174)
2024-11-24T04:32:18,042 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36545 is added to blk_1073741861_1037 (size=95174)
2024-11-24T04:32:18,047 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/.tmp/info/21e4e452e8d34499ad99a24d2b60a75d as hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/21e4e452e8d34499ad99a24d2b60a75d
2024-11-24T04:32:18,054 INFO  [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f22261b1394925c40b35fb57958000b/info of 3f22261b1394925c40b35fb57958000b into 21e4e452e8d34499ad99a24d2b60a75d(size=92.9 K), total size for store is 92.9 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-24T04:32:18,054 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f22261b1394925c40b35fb57958000b:

2024-11-24T04:32:18,054 INFO  [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732422723860.3f22261b1394925c40b35fb57958000b., storeName=3f22261b1394925c40b35fb57958000b/info, priority=13, startTime=1732422738026; duration=0sec
2024-11-24T04:32:18,054 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-24T04:32:18,054 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f22261b1394925c40b35fb57958000b:info
2024-11-24T04:32:18,608 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-24T04:32:18,858 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:18,858 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:19,859 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:19,859 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:20,859 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:20,859 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:21,860 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:21,860 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:22,860 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:22,860 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:23,861 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:23,861 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:24,862 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:24,862 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:25,862 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:25,862 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:26,863 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:26,863 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:27,863 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:27,863 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:28,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33299 {}] regionserver.HRegion(8855): Flush requested on 3f22261b1394925c40b35fb57958000b
2024-11-24T04:32:28,044 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3f22261b1394925c40b35fb57958000b 1/1 column families, dataSize=15.76 KB heapSize=17.13 KB
2024-11-24T04:32:28,049 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/.tmp/info/00b3823d03704189a5902dd17394217d is 1080, key is row0145/info:/1732422738003/Put/seqid=0
2024-11-24T04:32:28,053 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36545 is added to blk_1073741862_1038 (size=21156)
2024-11-24T04:32:28,053 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741862_1038 (size=21156)
2024-11-24T04:32:28,054 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.76 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/.tmp/info/00b3823d03704189a5902dd17394217d
2024-11-24T04:32:28,059 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/.tmp/info/00b3823d03704189a5902dd17394217d as hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/00b3823d03704189a5902dd17394217d
2024-11-24T04:32:28,063 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/00b3823d03704189a5902dd17394217d, entries=15, sequenceid=207, filesize=20.7 K
2024-11-24T04:32:28,064 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~15.76 KB/16140, heapSize ~17.11 KB/17520, currentSize=1.05 KB/1076 for 3f22261b1394925c40b35fb57958000b in 20ms, sequenceid=207, compaction requested=false
2024-11-24T04:32:28,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3f22261b1394925c40b35fb57958000b:

2024-11-24T04:32:28,864 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:28,864 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:29,587 INFO  [master/fadb2fe0ba32:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-11-24T04:32:29,587 INFO  [master/fadb2fe0ba32:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-11-24T04:32:29,864 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:29,864 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:30,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33299 {}] regionserver.HRegion(8855): Flush requested on 3f22261b1394925c40b35fb57958000b
2024-11-24T04:32:30,056 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3f22261b1394925c40b35fb57958000b 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-24T04:32:30,060 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/.tmp/info/6102229824444801a0371e4977aa6cd1 is 1080, key is row0160/info:/1732422748046/Put/seqid=0
2024-11-24T04:32:30,068 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741863_1039 (size=12516)
2024-11-24T04:32:30,068 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36545 is added to blk_1073741863_1039 (size=12516)
2024-11-24T04:32:30,069 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/.tmp/info/6102229824444801a0371e4977aa6cd1
2024-11-24T04:32:30,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/.tmp/info/6102229824444801a0371e4977aa6cd1 as hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/6102229824444801a0371e4977aa6cd1
2024-11-24T04:32:30,080 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/6102229824444801a0371e4977aa6cd1, entries=7, sequenceid=217, filesize=12.2 K
2024-11-24T04:32:30,080 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=16.81 KB/17216 for 3f22261b1394925c40b35fb57958000b in 24ms, sequenceid=217, compaction requested=true
2024-11-24T04:32:30,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3f22261b1394925c40b35fb57958000b:

2024-11-24T04:32:30,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f22261b1394925c40b35fb57958000b:info, priority=-2147483648, current under compaction store size is 1
2024-11-24T04:32:30,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-24T04:32:30,081 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-24T04:32:30,082 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 128846 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-24T04:32:30,082 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.HStore(1541): 3f22261b1394925c40b35fb57958000b/info is initiating minor compaction (all files)
2024-11-24T04:32:30,082 INFO  [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f22261b1394925c40b35fb57958000b/info in TestLogRolling-testLogRolling,row0062,1732422723860.3f22261b1394925c40b35fb57958000b.
2024-11-24T04:32:30,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33299 {}] regionserver.HRegion(8855): Flush requested on 3f22261b1394925c40b35fb57958000b
2024-11-24T04:32:30,082 INFO  [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/21e4e452e8d34499ad99a24d2b60a75d, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/00b3823d03704189a5902dd17394217d, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/6102229824444801a0371e4977aa6cd1] into tmpdir=hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/.tmp, totalSize=125.8 K
2024-11-24T04:32:30,082 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3f22261b1394925c40b35fb57958000b 1/1 column families, dataSize=18.91 KB heapSize=20.50 KB
2024-11-24T04:32:30,083 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21e4e452e8d34499ad99a24d2b60a75d, keycount=83, bloomtype=ROW, size=92.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1732422721783
2024-11-24T04:32:30,083 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00b3823d03704189a5902dd17394217d, keycount=15, bloomtype=ROW, size=20.7 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1732422738003
2024-11-24T04:32:30,083 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6102229824444801a0371e4977aa6cd1, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1732422748046
2024-11-24T04:32:30,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/.tmp/info/02d8eadad0dd4d198cb919feae57fd07 is 1080, key is row0167/info:/1732422750057/Put/seqid=0
2024-11-24T04:32:30,090 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741864_1040 (size=24394)
2024-11-24T04:32:30,091 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36545 is added to blk_1073741864_1040 (size=24394)
2024-11-24T04:32:30,091 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=18.91 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/.tmp/info/02d8eadad0dd4d198cb919feae57fd07
2024-11-24T04:32:30,096 INFO  [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f22261b1394925c40b35fb57958000b#info#compaction#79 average throughput is 35.92 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-24T04:32:30,096 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/.tmp/info/f15de4693fc14823ac744e91afd78c97 is 1080, key is row0062/info:/1732422721783/Put/seqid=0
2024-11-24T04:32:30,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/.tmp/info/02d8eadad0dd4d198cb919feae57fd07 as hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/02d8eadad0dd4d198cb919feae57fd07
2024-11-24T04:32:30,099 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741865_1041 (size=118996)
2024-11-24T04:32:30,100 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36545 is added to blk_1073741865_1041 (size=118996)
2024-11-24T04:32:30,102 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/02d8eadad0dd4d198cb919feae57fd07, entries=18, sequenceid=238, filesize=23.8 K
2024-11-24T04:32:30,103 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~18.91 KB/19368, heapSize ~20.48 KB/20976, currentSize=8.41 KB/8608 for 3f22261b1394925c40b35fb57958000b in 21ms, sequenceid=238, compaction requested=false
2024-11-24T04:32:30,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3f22261b1394925c40b35fb57958000b:

2024-11-24T04:32:30,104 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/.tmp/info/f15de4693fc14823ac744e91afd78c97 as hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/f15de4693fc14823ac744e91afd78c97
2024-11-24T04:32:30,110 INFO  [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f22261b1394925c40b35fb57958000b/info of 3f22261b1394925c40b35fb57958000b into f15de4693fc14823ac744e91afd78c97(size=116.2 K), total size for store is 140.0 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-24T04:32:30,110 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f22261b1394925c40b35fb57958000b:

2024-11-24T04:32:30,110 INFO  [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732422723860.3f22261b1394925c40b35fb57958000b., storeName=3f22261b1394925c40b35fb57958000b/info, priority=13, startTime=1732422750081; duration=0sec
2024-11-24T04:32:30,110 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-24T04:32:30,110 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f22261b1394925c40b35fb57958000b:info
2024-11-24T04:32:30,865 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:30,865 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:31,865 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:31,865 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:32,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33299 {}] regionserver.HRegion(8855): Flush requested on 3f22261b1394925c40b35fb57958000b
2024-11-24T04:32:32,099 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3f22261b1394925c40b35fb57958000b 1/1 column families, dataSize=9.46 KB heapSize=10.38 KB
2024-11-24T04:32:32,104 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/.tmp/info/6978d918c520408eadda4163475c7e61 is 1080, key is row0185/info:/1732422750083/Put/seqid=0
2024-11-24T04:32:32,109 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36545 is added to blk_1073741866_1042 (size=14672)
2024-11-24T04:32:32,109 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741866_1042 (size=14672)
2024-11-24T04:32:32,110 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=9.46 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/.tmp/info/6978d918c520408eadda4163475c7e61
2024-11-24T04:32:32,115 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/.tmp/info/6978d918c520408eadda4163475c7e61 as hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/6978d918c520408eadda4163475c7e61
2024-11-24T04:32:32,119 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/6978d918c520408eadda4163475c7e61, entries=9, sequenceid=251, filesize=14.3 K
2024-11-24T04:32:32,121 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~9.46 KB/9684, heapSize ~10.36 KB/10608, currentSize=12.61 KB/12912 for 3f22261b1394925c40b35fb57958000b in 21ms, sequenceid=251, compaction requested=true
2024-11-24T04:32:32,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3f22261b1394925c40b35fb57958000b:

2024-11-24T04:32:32,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f22261b1394925c40b35fb57958000b:info, priority=-2147483648, current under compaction store size is 1
2024-11-24T04:32:32,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-24T04:32:32,121 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-24T04:32:32,122 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 158062 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-24T04:32:32,122 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.HStore(1541): 3f22261b1394925c40b35fb57958000b/info is initiating minor compaction (all files)
2024-11-24T04:32:32,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33299 {}] regionserver.HRegion(8855): Flush requested on 3f22261b1394925c40b35fb57958000b
2024-11-24T04:32:32,122 INFO  [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f22261b1394925c40b35fb57958000b/info in TestLogRolling-testLogRolling,row0062,1732422723860.3f22261b1394925c40b35fb57958000b.
2024-11-24T04:32:32,122 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3f22261b1394925c40b35fb57958000b 1/1 column families, dataSize=14.71 KB heapSize=16 KB
2024-11-24T04:32:32,122 INFO  [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/f15de4693fc14823ac744e91afd78c97, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/02d8eadad0dd4d198cb919feae57fd07, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/6978d918c520408eadda4163475c7e61] into tmpdir=hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/.tmp, totalSize=154.4 K
2024-11-24T04:32:32,123 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] compactions.Compactor(225): Compacting f15de4693fc14823ac744e91afd78c97, keycount=105, bloomtype=ROW, size=116.2 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1732422721783
2024-11-24T04:32:32,124 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02d8eadad0dd4d198cb919feae57fd07, keycount=18, bloomtype=ROW, size=23.8 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732422750057
2024-11-24T04:32:32,124 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6978d918c520408eadda4163475c7e61, keycount=9, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732422750083
2024-11-24T04:32:32,127 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/.tmp/info/c672d85f1c2e4681bbe3b2c71e8fef81 is 1080, key is row0194/info:/1732422752101/Put/seqid=0
2024-11-24T04:32:32,132 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36545 is added to blk_1073741867_1043 (size=20089)
2024-11-24T04:32:32,133 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741867_1043 (size=20089)
2024-11-24T04:32:32,133 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.71 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/.tmp/info/c672d85f1c2e4681bbe3b2c71e8fef81
2024-11-24T04:32:32,139 INFO  [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f22261b1394925c40b35fb57958000b#info#compaction#82 average throughput is 45.15 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-24T04:32:32,139 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/.tmp/info/2f3812a54ae344188aeebed8636ac133 is 1080, key is row0062/info:/1732422721783/Put/seqid=0
2024-11-24T04:32:32,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/.tmp/info/c672d85f1c2e4681bbe3b2c71e8fef81 as hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/c672d85f1c2e4681bbe3b2c71e8fef81
2024-11-24T04:32:32,144 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/c672d85f1c2e4681bbe3b2c71e8fef81, entries=14, sequenceid=268, filesize=19.6 K
2024-11-24T04:32:32,145 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~14.71 KB/15064, heapSize ~15.98 KB/16368, currentSize=11.56 KB/11836 for 3f22261b1394925c40b35fb57958000b in 23ms, sequenceid=268, compaction requested=false
2024-11-24T04:32:32,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3f22261b1394925c40b35fb57958000b:

2024-11-24T04:32:32,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33299 {}] regionserver.HRegion(8855): Flush requested on 3f22261b1394925c40b35fb57958000b
2024-11-24T04:32:32,146 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3f22261b1394925c40b35fb57958000b 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-11-24T04:32:32,147 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36545 is added to blk_1073741868_1044 (size=148409)
2024-11-24T04:32:32,147 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741868_1044 (size=148409)
2024-11-24T04:32:32,151 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/.tmp/info/22ab83043d4841e0a3fcf3ab5b6a6d49 is 1080, key is row0208/info:/1732422752123/Put/seqid=0
2024-11-24T04:32:32,154 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/.tmp/info/2f3812a54ae344188aeebed8636ac133 as hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/2f3812a54ae344188aeebed8636ac133
2024-11-24T04:32:32,156 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36545 is added to blk_1073741869_1045 (size=17918)
2024-11-24T04:32:32,157 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741869_1045 (size=17918)
2024-11-24T04:32:32,157 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/.tmp/info/22ab83043d4841e0a3fcf3ab5b6a6d49
2024-11-24T04:32:32,161 INFO  [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f22261b1394925c40b35fb57958000b/info of 3f22261b1394925c40b35fb57958000b into 2f3812a54ae344188aeebed8636ac133(size=144.9 K), total size for store is 164.5 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-24T04:32:32,161 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f22261b1394925c40b35fb57958000b:

2024-11-24T04:32:32,161 INFO  [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732422723860.3f22261b1394925c40b35fb57958000b., storeName=3f22261b1394925c40b35fb57958000b/info, priority=13, startTime=1732422752121; duration=0sec
2024-11-24T04:32:32,161 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-24T04:32:32,161 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f22261b1394925c40b35fb57958000b:info
2024-11-24T04:32:32,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/.tmp/info/22ab83043d4841e0a3fcf3ab5b6a6d49 as hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/22ab83043d4841e0a3fcf3ab5b6a6d49
2024-11-24T04:32:32,167 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/22ab83043d4841e0a3fcf3ab5b6a6d49, entries=12, sequenceid=283, filesize=17.5 K
2024-11-24T04:32:32,168 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=5.25 KB/5380 for 3f22261b1394925c40b35fb57958000b in 23ms, sequenceid=283, compaction requested=true
2024-11-24T04:32:32,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3f22261b1394925c40b35fb57958000b:

2024-11-24T04:32:32,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f22261b1394925c40b35fb57958000b:info, priority=-2147483648, current under compaction store size is 1
2024-11-24T04:32:32,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-24T04:32:32,168 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-24T04:32:32,169 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 186416 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-24T04:32:32,169 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.HStore(1541): 3f22261b1394925c40b35fb57958000b/info is initiating minor compaction (all files)
2024-11-24T04:32:32,169 INFO  [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f22261b1394925c40b35fb57958000b/info in TestLogRolling-testLogRolling,row0062,1732422723860.3f22261b1394925c40b35fb57958000b.
2024-11-24T04:32:32,169 INFO  [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/2f3812a54ae344188aeebed8636ac133, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/c672d85f1c2e4681bbe3b2c71e8fef81, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/22ab83043d4841e0a3fcf3ab5b6a6d49] into tmpdir=hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/.tmp, totalSize=182.0 K
2024-11-24T04:32:32,170 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f3812a54ae344188aeebed8636ac133, keycount=132, bloomtype=ROW, size=144.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732422721783
2024-11-24T04:32:32,170 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] compactions.Compactor(225): Compacting c672d85f1c2e4681bbe3b2c71e8fef81, keycount=14, bloomtype=ROW, size=19.6 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1732422752101
2024-11-24T04:32:32,170 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22ab83043d4841e0a3fcf3ab5b6a6d49, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1732422752123
2024-11-24T04:32:32,181 INFO  [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f22261b1394925c40b35fb57958000b#info#compaction#84 average throughput is 54.04 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-24T04:32:32,181 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/.tmp/info/9357c7fea1f34500976d9c538ed617c3 is 1080, key is row0062/info:/1732422721783/Put/seqid=0
2024-11-24T04:32:32,184 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741870_1046 (size=176570)
2024-11-24T04:32:32,185 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36545 is added to blk_1073741870_1046 (size=176570)
2024-11-24T04:32:32,190 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/.tmp/info/9357c7fea1f34500976d9c538ed617c3 as hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/9357c7fea1f34500976d9c538ed617c3
2024-11-24T04:32:32,195 INFO  [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f22261b1394925c40b35fb57958000b/info of 3f22261b1394925c40b35fb57958000b into 9357c7fea1f34500976d9c538ed617c3(size=172.4 K), total size for store is 172.4 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-24T04:32:32,195 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f22261b1394925c40b35fb57958000b:

2024-11-24T04:32:32,195 INFO  [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732422723860.3f22261b1394925c40b35fb57958000b., storeName=3f22261b1394925c40b35fb57958000b/info, priority=13, startTime=1732422752168; duration=0sec
2024-11-24T04:32:32,195 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-24T04:32:32,195 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f22261b1394925c40b35fb57958000b:info
2024-11-24T04:32:32,866 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:32,866 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:33,866 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:33,866 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:34,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33299 {}] regionserver.HRegion(8855): Flush requested on 3f22261b1394925c40b35fb57958000b
2024-11-24T04:32:34,159 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3f22261b1394925c40b35fb57958000b 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-24T04:32:34,163 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/.tmp/info/3d71eea11c7a4670aedd2b00b0808d6e is 1080, key is row0220/info:/1732422752147/Put/seqid=0
2024-11-24T04:32:34,169 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741871_1047 (size=12523)
2024-11-24T04:32:34,170 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36545 is added to blk_1073741871_1047 (size=12523)
2024-11-24T04:32:34,170 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/.tmp/info/3d71eea11c7a4670aedd2b00b0808d6e
2024-11-24T04:32:34,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/.tmp/info/3d71eea11c7a4670aedd2b00b0808d6e as hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/3d71eea11c7a4670aedd2b00b0808d6e
2024-11-24T04:32:34,179 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/3d71eea11c7a4670aedd2b00b0808d6e, entries=7, sequenceid=295, filesize=12.2 K
2024-11-24T04:32:34,180 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=15.76 KB/16140 for 3f22261b1394925c40b35fb57958000b in 21ms, sequenceid=295, compaction requested=false
2024-11-24T04:32:34,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3f22261b1394925c40b35fb57958000b:

2024-11-24T04:32:34,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33299 {}] regionserver.HRegion(8855): Flush requested on 3f22261b1394925c40b35fb57958000b
2024-11-24T04:32:34,181 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3f22261b1394925c40b35fb57958000b 1/1 column families, dataSize=16.81 KB heapSize=18.25 KB
2024-11-24T04:32:34,185 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/.tmp/info/92285da4066e40f680b7bf1612807fc3 is 1080, key is row0227/info:/1732422754160/Put/seqid=0
2024-11-24T04:32:34,190 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741872_1048 (size=22254)
2024-11-24T04:32:34,190 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36545 is added to blk_1073741872_1048 (size=22254)
2024-11-24T04:32:34,190 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.81 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/.tmp/info/92285da4066e40f680b7bf1612807fc3
2024-11-24T04:32:34,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/.tmp/info/92285da4066e40f680b7bf1612807fc3 as hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/92285da4066e40f680b7bf1612807fc3
2024-11-24T04:32:34,200 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/92285da4066e40f680b7bf1612807fc3, entries=16, sequenceid=314, filesize=21.7 K
2024-11-24T04:32:34,200 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33299 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit.
org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=3f22261b1394925c40b35fb57958000b, server=fadb2fe0ba32,33299,1732422708667
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT]
2024-11-24T04:32:34,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33299 {}] ipc.CallRunner(138): callId: 265 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:46948 deadline: 1732422764200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=3f22261b1394925c40b35fb57958000b, server=fadb2fe0ba32,33299,1732422708667
2024-11-24T04:32:34,201 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestLogRolling-testLogRolling,row0062,1732422723860.3f22261b1394925c40b35fb57958000b., hostname=fadb2fe0ba32,33299,1732422708667, seqNum=127 , the old value is region=TestLogRolling-testLogRolling,row0062,1732422723860.3f22261b1394925c40b35fb57958000b., hostname=fadb2fe0ba32,33299,1732422708667, seqNum=127, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=3f22261b1394925c40b35fb57958000b, server=fadb2fe0ba32,33299,1732422708667
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308)
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329)
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216)
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-11-24T04:32:34,201 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestLogRolling-testLogRolling,row0062,1732422723860.3f22261b1394925c40b35fb57958000b., hostname=fadb2fe0ba32,33299,1732422708667, seqNum=127 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=3f22261b1394925c40b35fb57958000b, server=fadb2fe0ba32,33299,1732422708667
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308)
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329)
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216)
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-11-24T04:32:34,201 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~16.81 KB/17216, heapSize ~18.23 KB/18672, currentSize=13.66 KB/13988 for 3f22261b1394925c40b35fb57958000b in 20ms, sequenceid=314, compaction requested=true
2024-11-24T04:32:34,201 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestLogRolling-testLogRolling,row0062,1732422723860.3f22261b1394925c40b35fb57958000b., hostname=fadb2fe0ba32,33299,1732422708667, seqNum=127 because the exception is null or not the one we care about
2024-11-24T04:32:34,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3f22261b1394925c40b35fb57958000b:

2024-11-24T04:32:34,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3f22261b1394925c40b35fb57958000b:info, priority=-2147483648, current under compaction store size is 1
2024-11-24T04:32:34,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-24T04:32:34,202 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-24T04:32:34,203 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 211347 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-24T04:32:34,203 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.HStore(1541): 3f22261b1394925c40b35fb57958000b/info is initiating minor compaction (all files)
2024-11-24T04:32:34,203 INFO  [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3f22261b1394925c40b35fb57958000b/info in TestLogRolling-testLogRolling,row0062,1732422723860.3f22261b1394925c40b35fb57958000b.
2024-11-24T04:32:34,203 INFO  [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/9357c7fea1f34500976d9c538ed617c3, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/3d71eea11c7a4670aedd2b00b0808d6e, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/92285da4066e40f680b7bf1612807fc3] into tmpdir=hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/.tmp, totalSize=206.4 K
2024-11-24T04:32:34,203 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9357c7fea1f34500976d9c538ed617c3, keycount=158, bloomtype=ROW, size=172.4 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1732422721783
2024-11-24T04:32:34,203 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d71eea11c7a4670aedd2b00b0808d6e, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732422752147
2024-11-24T04:32:34,204 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92285da4066e40f680b7bf1612807fc3, keycount=16, bloomtype=ROW, size=21.7 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1732422754160
2024-11-24T04:32:34,213 INFO  [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3f22261b1394925c40b35fb57958000b#info#compaction#87 average throughput is 92.87 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-24T04:32:34,214 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/.tmp/info/f414676546ec493689520f196e4c424a is 1080, key is row0062/info:/1732422721783/Put/seqid=0
2024-11-24T04:32:34,217 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741873_1049 (size=201493)
2024-11-24T04:32:34,217 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36545 is added to blk_1073741873_1049 (size=201493)
2024-11-24T04:32:34,222 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/.tmp/info/f414676546ec493689520f196e4c424a as hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/f414676546ec493689520f196e4c424a
2024-11-24T04:32:34,228 INFO  [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3f22261b1394925c40b35fb57958000b/info of 3f22261b1394925c40b35fb57958000b into f414676546ec493689520f196e4c424a(size=196.8 K), total size for store is 196.8 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-24T04:32:34,228 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3f22261b1394925c40b35fb57958000b:

2024-11-24T04:32:34,228 INFO  [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732422723860.3f22261b1394925c40b35fb57958000b., storeName=3f22261b1394925c40b35fb57958000b/info, priority=13, startTime=1732422754202; duration=0sec
2024-11-24T04:32:34,228 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-24T04:32:34,228 DEBUG [RS:0;fadb2fe0ba32:33299-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3f22261b1394925c40b35fb57958000b:info
2024-11-24T04:32:34,447 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 20375
2024-11-24T04:32:34,867 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:34,867 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:35,868 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:35,868 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:36,868 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:36,868 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:37,869 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:37,869 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:38,869 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:38,869 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:39,870 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:39,870 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:40,870 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:40,870 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:41,871 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:41,871 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:42,871 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:42,871 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:43,872 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:43,872 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:44,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33299 {}] regionserver.HRegion(8855): Flush requested on 3f22261b1394925c40b35fb57958000b
2024-11-24T04:32:44,224 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3f22261b1394925c40b35fb57958000b 1/1 column families, dataSize=14.71 KB heapSize=16 KB
2024-11-24T04:32:44,229 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/.tmp/info/7646504b12124e61acff97088f11a552 is 1080, key is row0243/info:/1732422754182/Put/seqid=0
2024-11-24T04:32:44,233 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36545 is added to blk_1073741874_1050 (size=20092)
2024-11-24T04:32:44,233 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741874_1050 (size=20092)
2024-11-24T04:32:44,234 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.71 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/.tmp/info/7646504b12124e61acff97088f11a552
2024-11-24T04:32:44,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/.tmp/info/7646504b12124e61acff97088f11a552 as hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/7646504b12124e61acff97088f11a552
2024-11-24T04:32:44,244 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/7646504b12124e61acff97088f11a552, entries=14, sequenceid=332, filesize=19.6 K
2024-11-24T04:32:44,245 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~14.71 KB/15064, heapSize ~15.98 KB/16368, currentSize=0 B/0 for 3f22261b1394925c40b35fb57958000b in 21ms, sequenceid=332, compaction requested=false
2024-11-24T04:32:44,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3f22261b1394925c40b35fb57958000b:

2024-11-24T04:32:44,873 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:44,873 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:45,873 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:45,873 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:46,225 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(285): after writing there are 0 log files
2024-11-24T04:32:46,225 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor fadb2fe0ba32%2C33299%2C1732422708667.1732422766225
2024-11-24T04:32:46,231 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:32:46,231 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:32:46,231 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:32:46,231 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:32:46,231 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:32:46,231 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/WALs/fadb2fe0ba32,33299,1732422708667/fadb2fe0ba32%2C33299%2C1732422708667.1732422709048 with entries=312, filesize=308.51 KB; new WAL /user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/WALs/fadb2fe0ba32,33299,1732422708667/fadb2fe0ba32%2C33299%2C1732422708667.1732422766225
2024-11-24T04:32:46,232 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40733:40733),(127.0.0.1/127.0.0.1:36251:36251)]
2024-11-24T04:32:46,232 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/WALs/fadb2fe0ba32,33299,1732422708667/fadb2fe0ba32%2C33299%2C1732422708667.1732422709048 is not closed yet, will try archiving it next time
2024-11-24T04:32:46,233 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36545 is added to blk_1073741833_1009 (size=315921)
2024-11-24T04:32:46,233 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741833_1009 (size=315921)
2024-11-24T04:32:46,236 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 3f22261b1394925c40b35fb57958000b:

2024-11-24T04:32:46,236 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 9449b123c00b1a221a275bb41283b414:

2024-11-24T04:32:46,237 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=670 B heapSize=2.02 KB
2024-11-24T04:32:46,237 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/WALs/fadb2fe0ba32,33299,1732422708667/fadb2fe0ba32%2C33299%2C1732422708667.1732422709048 to hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/oldWALs/fadb2fe0ba32%2C33299%2C1732422708667.1732422709048
2024-11-24T04:32:46,240 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/hbase/meta/1588230740/.tmp/info/869a5ab040ad4cf0a0d0c4963de5aa88 is 186, key is TestLogRolling-testLogRolling,,1732422723860.9449b123c00b1a221a275bb41283b414./info:regioninfo/1732422724563/Put/seqid=0
2024-11-24T04:32:46,244 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36545 is added to blk_1073741876_1052 (size=6153)
2024-11-24T04:32:46,244 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741876_1052 (size=6153)
2024-11-24T04:32:46,244 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=670 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/hbase/meta/1588230740/.tmp/info/869a5ab040ad4cf0a0d0c4963de5aa88
2024-11-24T04:32:46,249 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/hbase/meta/1588230740/.tmp/info/869a5ab040ad4cf0a0d0c4963de5aa88 as hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/hbase/meta/1588230740/info/869a5ab040ad4cf0a0d0c4963de5aa88
2024-11-24T04:32:46,253 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/hbase/meta/1588230740/info/869a5ab040ad4cf0a0d0c4963de5aa88, entries=5, sequenceid=21, filesize=6.0 K
2024-11-24T04:32:46,254 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~670 B/670, heapSize ~1.25 KB/1280, currentSize=0 B/0 for 1588230740 in 18ms, sequenceid=21, compaction requested=false
2024-11-24T04:32:46,254 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1588230740:

2024-11-24T04:32:46,255 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1])
2024-11-24T04:32:46,255 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-24T04:32:46,255 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-24T04:32:46,255 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-24T04:32:46,255 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T04:32:46,255 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T04:32:46,255 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-24T04:32:46,256 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-24T04:32:46,256 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=384597719, stopped=false
2024-11-24T04:32:46,256 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=fadb2fe0ba32,34999,1732422708624
2024-11-24T04:32:46,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34999-0x101483e98110000, quorum=127.0.0.1:64360, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-24T04:32:46,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33299-0x101483e98110001, quorum=127.0.0.1:64360, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-24T04:32:46,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33299-0x101483e98110001, quorum=127.0.0.1:64360, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:32:46,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34999-0x101483e98110000, quorum=127.0.0.1:64360, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:32:46,258 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-24T04:32:46,258 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-24T04:32:46,258 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:34999-0x101483e98110000, quorum=127.0.0.1:64360, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-24T04:32:46,258 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33299-0x101483e98110001, quorum=127.0.0.1:64360, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-24T04:32:46,258 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-24T04:32:46,259 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T04:32:46,259 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'fadb2fe0ba32,33299,1732422708667' *****
2024-11-24T04:32:46,259 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-24T04:32:46,259 INFO  [RS:0;fadb2fe0ba32:33299 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-24T04:32:46,259 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-24T04:32:46,259 INFO  [RS:0;fadb2fe0ba32:33299 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-24T04:32:46,259 INFO  [RS:0;fadb2fe0ba32:33299 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-24T04:32:46,260 INFO  [RS:0;fadb2fe0ba32:33299 {}] regionserver.HRegionServer(3091): Received CLOSE for 3f22261b1394925c40b35fb57958000b
2024-11-24T04:32:46,260 INFO  [RS:0;fadb2fe0ba32:33299 {}] regionserver.HRegionServer(3091): Received CLOSE for 9449b123c00b1a221a275bb41283b414
2024-11-24T04:32:46,260 INFO  [RS:0;fadb2fe0ba32:33299 {}] regionserver.HRegionServer(959): stopping server fadb2fe0ba32,33299,1732422708667
2024-11-24T04:32:46,260 INFO  [RS:0;fadb2fe0ba32:33299 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-24T04:32:46,260 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3f22261b1394925c40b35fb57958000b, disabling compactions & flushes
2024-11-24T04:32:46,260 INFO  [RS:0;fadb2fe0ba32:33299 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;fadb2fe0ba32:33299.
2024-11-24T04:32:46,260 INFO  [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,row0062,1732422723860.3f22261b1394925c40b35fb57958000b.
2024-11-24T04:32:46,260 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,row0062,1732422723860.3f22261b1394925c40b35fb57958000b.
2024-11-24T04:32:46,260 DEBUG [RS:0;fadb2fe0ba32:33299 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-24T04:32:46,260 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,row0062,1732422723860.3f22261b1394925c40b35fb57958000b. after waiting 0 ms
2024-11-24T04:32:46,260 DEBUG [RS:0;fadb2fe0ba32:33299 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T04:32:46,260 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,row0062,1732422723860.3f22261b1394925c40b35fb57958000b.
2024-11-24T04:32:46,260 INFO  [RS:0;fadb2fe0ba32:33299 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-24T04:32:46,260 INFO  [RS:0;fadb2fe0ba32:33299 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-24T04:32:46,260 INFO  [RS:0;fadb2fe0ba32:33299 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-24T04:32:46,260 INFO  [RS:0;fadb2fe0ba32:33299 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-24T04:32:46,260 INFO  [RS:0;fadb2fe0ba32:33299 {}] regionserver.HRegionServer(1321): Waiting on 3 regions to close
2024-11-24T04:32:46,260 DEBUG [RS:0;fadb2fe0ba32:33299 {}] regionserver.HRegionServer(1325): Online Regions={3f22261b1394925c40b35fb57958000b=TestLogRolling-testLogRolling,row0062,1732422723860.3f22261b1394925c40b35fb57958000b., 9449b123c00b1a221a275bb41283b414=TestLogRolling-testLogRolling,,1732422723860.9449b123c00b1a221a275bb41283b414., 1588230740=hbase:meta,,1.1588230740}
2024-11-24T04:32:46,260 DEBUG [RS:0;fadb2fe0ba32:33299 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 3f22261b1394925c40b35fb57958000b, 9449b123c00b1a221a275bb41283b414
2024-11-24T04:32:46,260 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-24T04:32:46,261 INFO  [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-24T04:32:46,261 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-24T04:32:46,261 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-24T04:32:46,261 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-24T04:32:46,260 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732422723860.3f22261b1394925c40b35fb57958000b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/005ca3e2be6042eab9bd6f6c772b9469.a96be4dd9305efb471150cc3d77a7019->hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/info/005ca3e2be6042eab9bd6f6c772b9469-top, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/TestLogRolling-testLogRolling=a96be4dd9305efb471150cc3d77a7019-6df4256bb0f1422cac4b53f1647f7eaa, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/TestLogRolling-testLogRolling=a96be4dd9305efb471150cc3d77a7019-dda9939d8dc9423abdc85e4d4095a78f, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/55e010b042dc44d0a08b905cda87de25, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/TestLogRolling-testLogRolling=a96be4dd9305efb471150cc3d77a7019-44a0bbc0bc374b0282c036160c35f7c3, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/71241d62e1264a959e1b61319644b704, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/d062343cbbd34e2c992e85c468d9976f, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/e951e9e1b9214854a0f6023fb3afc0e8, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/b1c4842d369d499ba01af0994f04b0b0, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/21e4e452e8d34499ad99a24d2b60a75d, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/9fd4851a42f641f1b6f934635d604f43, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/00b3823d03704189a5902dd17394217d, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/f15de4693fc14823ac744e91afd78c97, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/6102229824444801a0371e4977aa6cd1, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/02d8eadad0dd4d198cb919feae57fd07, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/2f3812a54ae344188aeebed8636ac133, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/6978d918c520408eadda4163475c7e61, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/c672d85f1c2e4681bbe3b2c71e8fef81, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/9357c7fea1f34500976d9c538ed617c3, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/22ab83043d4841e0a3fcf3ab5b6a6d49, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/3d71eea11c7a4670aedd2b00b0808d6e, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/92285da4066e40f680b7bf1612807fc3] to archive
2024-11-24T04:32:46,262 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732422723860.3f22261b1394925c40b35fb57958000b.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-24T04:32:46,263 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732422723860.3f22261b1394925c40b35fb57958000b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/005ca3e2be6042eab9bd6f6c772b9469.a96be4dd9305efb471150cc3d77a7019 to hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/archive/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/005ca3e2be6042eab9bd6f6c772b9469.a96be4dd9305efb471150cc3d77a7019
2024-11-24T04:32:46,265 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732422723860.3f22261b1394925c40b35fb57958000b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/TestLogRolling-testLogRolling=a96be4dd9305efb471150cc3d77a7019-6df4256bb0f1422cac4b53f1647f7eaa to hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/archive/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/TestLogRolling-testLogRolling=a96be4dd9305efb471150cc3d77a7019-6df4256bb0f1422cac4b53f1647f7eaa
2024-11-24T04:32:46,265 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1
2024-11-24T04:32:46,265 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-24T04:32:46,265 INFO  [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-24T04:32:46,265 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732422766260Running coprocessor pre-close hooks at 1732422766260Disabling compacts and flushes for region at 1732422766260Disabling writes for close at 1732422766261 (+1 ms)Writing region close event to WAL at 1732422766262 (+1 ms)Running coprocessor post-close hooks at 1732422766265 (+3 ms)Closed at 1732422766265
2024-11-24T04:32:46,266 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-24T04:32:46,266 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732422723860.3f22261b1394925c40b35fb57958000b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/TestLogRolling-testLogRolling=a96be4dd9305efb471150cc3d77a7019-dda9939d8dc9423abdc85e4d4095a78f to hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/archive/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/TestLogRolling-testLogRolling=a96be4dd9305efb471150cc3d77a7019-dda9939d8dc9423abdc85e4d4095a78f
2024-11-24T04:32:46,266 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-24T04:32:46,266 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-24T04:32:46,267 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732422723860.3f22261b1394925c40b35fb57958000b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/55e010b042dc44d0a08b905cda87de25 to hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/archive/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/55e010b042dc44d0a08b905cda87de25
2024-11-24T04:32:46,268 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732422723860.3f22261b1394925c40b35fb57958000b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/TestLogRolling-testLogRolling=a96be4dd9305efb471150cc3d77a7019-44a0bbc0bc374b0282c036160c35f7c3 to hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/archive/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/TestLogRolling-testLogRolling=a96be4dd9305efb471150cc3d77a7019-44a0bbc0bc374b0282c036160c35f7c3
2024-11-24T04:32:46,269 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732422723860.3f22261b1394925c40b35fb57958000b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/71241d62e1264a959e1b61319644b704 to hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/archive/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/71241d62e1264a959e1b61319644b704
2024-11-24T04:32:46,270 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732422723860.3f22261b1394925c40b35fb57958000b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/d062343cbbd34e2c992e85c468d9976f to hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/archive/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/d062343cbbd34e2c992e85c468d9976f
2024-11-24T04:32:46,271 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732422723860.3f22261b1394925c40b35fb57958000b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/e951e9e1b9214854a0f6023fb3afc0e8 to hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/archive/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/e951e9e1b9214854a0f6023fb3afc0e8
2024-11-24T04:32:46,272 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732422723860.3f22261b1394925c40b35fb57958000b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/b1c4842d369d499ba01af0994f04b0b0 to hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/archive/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/b1c4842d369d499ba01af0994f04b0b0
2024-11-24T04:32:46,273 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732422723860.3f22261b1394925c40b35fb57958000b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/21e4e452e8d34499ad99a24d2b60a75d to hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/archive/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/21e4e452e8d34499ad99a24d2b60a75d
2024-11-24T04:32:46,274 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732422723860.3f22261b1394925c40b35fb57958000b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/9fd4851a42f641f1b6f934635d604f43 to hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/archive/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/9fd4851a42f641f1b6f934635d604f43
2024-11-24T04:32:46,275 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732422723860.3f22261b1394925c40b35fb57958000b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/00b3823d03704189a5902dd17394217d to hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/archive/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/00b3823d03704189a5902dd17394217d
2024-11-24T04:32:46,276 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732422723860.3f22261b1394925c40b35fb57958000b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/f15de4693fc14823ac744e91afd78c97 to hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/archive/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/f15de4693fc14823ac744e91afd78c97
2024-11-24T04:32:46,277 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732422723860.3f22261b1394925c40b35fb57958000b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/6102229824444801a0371e4977aa6cd1 to hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/archive/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/6102229824444801a0371e4977aa6cd1
2024-11-24T04:32:46,278 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732422723860.3f22261b1394925c40b35fb57958000b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/02d8eadad0dd4d198cb919feae57fd07 to hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/archive/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/02d8eadad0dd4d198cb919feae57fd07
2024-11-24T04:32:46,279 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732422723860.3f22261b1394925c40b35fb57958000b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/2f3812a54ae344188aeebed8636ac133 to hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/archive/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/2f3812a54ae344188aeebed8636ac133
2024-11-24T04:32:46,280 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732422723860.3f22261b1394925c40b35fb57958000b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/6978d918c520408eadda4163475c7e61 to hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/archive/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/6978d918c520408eadda4163475c7e61
2024-11-24T04:32:46,281 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732422723860.3f22261b1394925c40b35fb57958000b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/c672d85f1c2e4681bbe3b2c71e8fef81 to hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/archive/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/c672d85f1c2e4681bbe3b2c71e8fef81
2024-11-24T04:32:46,282 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732422723860.3f22261b1394925c40b35fb57958000b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/9357c7fea1f34500976d9c538ed617c3 to hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/archive/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/9357c7fea1f34500976d9c538ed617c3
2024-11-24T04:32:46,283 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732422723860.3f22261b1394925c40b35fb57958000b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/22ab83043d4841e0a3fcf3ab5b6a6d49 to hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/archive/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/22ab83043d4841e0a3fcf3ab5b6a6d49
2024-11-24T04:32:46,284 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732422723860.3f22261b1394925c40b35fb57958000b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/3d71eea11c7a4670aedd2b00b0808d6e to hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/archive/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/3d71eea11c7a4670aedd2b00b0808d6e
2024-11-24T04:32:46,285 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732422723860.3f22261b1394925c40b35fb57958000b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/92285da4066e40f680b7bf1612807fc3 to hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/archive/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/info/92285da4066e40f680b7bf1612807fc3
2024-11-24T04:32:46,285 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732422723860.3f22261b1394925c40b35fb57958000b.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=fadb2fe0ba32:34999 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 16 more
2024-11-24T04:32:46,286 WARN  [StoreCloser-TestLogRolling-testLogRolling,row0062,1732422723860.3f22261b1394925c40b35fb57958000b.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [55e010b042dc44d0a08b905cda87de25=43081, 71241d62e1264a959e1b61319644b704=12516, d062343cbbd34e2c992e85c468d9976f=71283, e951e9e1b9214854a0f6023fb3afc0e8=25472, b1c4842d369d499ba01af0994f04b0b0=12516, 21e4e452e8d34499ad99a24d2b60a75d=95174, 9fd4851a42f641f1b6f934635d604f43=21156, 00b3823d03704189a5902dd17394217d=21156, f15de4693fc14823ac744e91afd78c97=118996, 6102229824444801a0371e4977aa6cd1=12516, 02d8eadad0dd4d198cb919feae57fd07=24394, 2f3812a54ae344188aeebed8636ac133=148409, 6978d918c520408eadda4163475c7e61=14672, c672d85f1c2e4681bbe3b2c71e8fef81=20089, 9357c7fea1f34500976d9c538ed617c3=176570, 22ab83043d4841e0a3fcf3ab5b6a6d49=17918, 3d71eea11c7a4670aedd2b00b0808d6e=12523, 92285da4066e40f680b7bf1612807fc3=22254]
2024-11-24T04:32:46,289 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/3f22261b1394925c40b35fb57958000b/recovered.edits/336.seqid, newMaxSeqId=336, maxSeqId=126
2024-11-24T04:32:46,289 INFO  [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,row0062,1732422723860.3f22261b1394925c40b35fb57958000b.
2024-11-24T04:32:46,289 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3f22261b1394925c40b35fb57958000b:
Waiting for close lock at 1732422766260Running coprocessor pre-close hooks at 1732422766260Disabling compacts and flushes for region at 1732422766260Disabling writes for close at 1732422766260Writing region close event to WAL at 1732422766286 (+26 ms)Running coprocessor post-close hooks at 1732422766289 (+3 ms)Closed at 1732422766289
2024-11-24T04:32:46,289 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,row0062,1732422723860.3f22261b1394925c40b35fb57958000b.
2024-11-24T04:32:46,290 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9449b123c00b1a221a275bb41283b414, disabling compactions & flushes
2024-11-24T04:32:46,290 INFO  [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1732422723860.9449b123c00b1a221a275bb41283b414.
2024-11-24T04:32:46,290 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1732422723860.9449b123c00b1a221a275bb41283b414.
2024-11-24T04:32:46,290 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1732422723860.9449b123c00b1a221a275bb41283b414. after waiting 0 ms
2024-11-24T04:32:46,290 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1732422723860.9449b123c00b1a221a275bb41283b414.
2024-11-24T04:32:46,290 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732422723860.9449b123c00b1a221a275bb41283b414.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/9449b123c00b1a221a275bb41283b414/info/005ca3e2be6042eab9bd6f6c772b9469.a96be4dd9305efb471150cc3d77a7019->hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/a96be4dd9305efb471150cc3d77a7019/info/005ca3e2be6042eab9bd6f6c772b9469-bottom] to archive
2024-11-24T04:32:46,291 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732422723860.9449b123c00b1a221a275bb41283b414.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-24T04:32:46,292 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732422723860.9449b123c00b1a221a275bb41283b414.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/9449b123c00b1a221a275bb41283b414/info/005ca3e2be6042eab9bd6f6c772b9469.a96be4dd9305efb471150cc3d77a7019 to hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/archive/data/default/TestLogRolling-testLogRolling/9449b123c00b1a221a275bb41283b414/info/005ca3e2be6042eab9bd6f6c772b9469.a96be4dd9305efb471150cc3d77a7019
2024-11-24T04:32:46,292 WARN  [StoreCloser-TestLogRolling-testLogRolling,,1732422723860.9449b123c00b1a221a275bb41283b414.-1 {}] regionserver.HStore(2414): Failed to report archival of files: []
2024-11-24T04:32:46,295 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/data/default/TestLogRolling-testLogRolling/9449b123c00b1a221a275bb41283b414/recovered.edits/131.seqid, newMaxSeqId=131, maxSeqId=126
2024-11-24T04:32:46,295 INFO  [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1732422723860.9449b123c00b1a221a275bb41283b414.
2024-11-24T04:32:46,295 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9449b123c00b1a221a275bb41283b414:
Waiting for close lock at 1732422766290Running coprocessor pre-close hooks at 1732422766290Disabling compacts and flushes for region at 1732422766290Disabling writes for close at 1732422766290Writing region close event to WAL at 1732422766292 (+2 ms)Running coprocessor post-close hooks at 1732422766295 (+3 ms)Closed at 1732422766295
2024-11-24T04:32:46,295 DEBUG [RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,,1732422723860.9449b123c00b1a221a275bb41283b414.
2024-11-24T04:32:46,461 INFO  [RS:0;fadb2fe0ba32:33299 {}] regionserver.HRegionServer(976): stopping server fadb2fe0ba32,33299,1732422708667; all regions closed.
2024-11-24T04:32:46,461 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:32:46,461 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:32:46,461 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:32:46,461 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:32:46,462 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:32:46,463 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741834_1010 (size=8107)
2024-11-24T04:32:46,464 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36545 is added to blk_1073741834_1010 (size=8107)
2024-11-24T04:32:46,466 DEBUG [RS:0;fadb2fe0ba32:33299 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/oldWALs
2024-11-24T04:32:46,466 INFO  [RS:0;fadb2fe0ba32:33299 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog fadb2fe0ba32%2C33299%2C1732422708667.meta:.meta(num 1732422709437)
2024-11-24T04:32:46,466 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:32:46,466 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:32:46,466 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:32:46,466 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:32:46,466 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:32:46,467 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36545 is added to blk_1073741875_1051 (size=780)
2024-11-24T04:32:46,468 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741875_1051 (size=780)
2024-11-24T04:32:46,469 DEBUG [RS:0;fadb2fe0ba32:33299 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/oldWALs
2024-11-24T04:32:46,470 INFO  [RS:0;fadb2fe0ba32:33299 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog fadb2fe0ba32%2C33299%2C1732422708667:(num 1732422766225)
2024-11-24T04:32:46,470 DEBUG [RS:0;fadb2fe0ba32:33299 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T04:32:46,470 INFO  [RS:0;fadb2fe0ba32:33299 {}] regionserver.LeaseManager(133): Closed leases
2024-11-24T04:32:46,470 INFO  [RS:0;fadb2fe0ba32:33299 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-24T04:32:46,470 INFO  [RS:0;fadb2fe0ba32:33299 {}] hbase.ChoreService(370): Chore service for: regionserver/fadb2fe0ba32:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-24T04:32:46,470 INFO  [RS:0;fadb2fe0ba32:33299 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-24T04:32:46,470 INFO  [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-24T04:32:46,470 INFO  [RS:0;fadb2fe0ba32:33299 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33299
2024-11-24T04:32:46,472 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33299-0x101483e98110001, quorum=127.0.0.1:64360, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/fadb2fe0ba32,33299,1732422708667
2024-11-24T04:32:46,472 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34999-0x101483e98110000, quorum=127.0.0.1:64360, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-24T04:32:46,472 INFO  [RS:0;fadb2fe0ba32:33299 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-24T04:32:46,474 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [fadb2fe0ba32,33299,1732422708667]
2024-11-24T04:32:46,477 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/fadb2fe0ba32,33299,1732422708667 already deleted, retry=false
2024-11-24T04:32:46,477 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; fadb2fe0ba32,33299,1732422708667 expired; onlineServers=0
2024-11-24T04:32:46,477 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'fadb2fe0ba32,34999,1732422708624' *****
2024-11-24T04:32:46,477 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-24T04:32:46,477 INFO  [M:0;fadb2fe0ba32:34999 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-24T04:32:46,477 INFO  [M:0;fadb2fe0ba32:34999 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-24T04:32:46,478 DEBUG [M:0;fadb2fe0ba32:34999 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-24T04:32:46,478 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-24T04:32:46,478 DEBUG [M:0;fadb2fe0ba32:34999 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-24T04:32:46,478 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster-HFileCleaner.small.0-1732422708832 {}] cleaner.HFileCleaner(306): Exit Thread[master/fadb2fe0ba32:0:becomeActiveMaster-HFileCleaner.small.0-1732422708832,5,FailOnTimeoutGroup]
2024-11-24T04:32:46,478 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster-HFileCleaner.large.0-1732422708832 {}] cleaner.HFileCleaner(306): Exit Thread[master/fadb2fe0ba32:0:becomeActiveMaster-HFileCleaner.large.0-1732422708832,5,FailOnTimeoutGroup]
2024-11-24T04:32:46,478 INFO  [M:0;fadb2fe0ba32:34999 {}] hbase.ChoreService(370): Chore service for: master/fadb2fe0ba32:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-24T04:32:46,478 INFO  [M:0;fadb2fe0ba32:34999 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-24T04:32:46,478 DEBUG [M:0;fadb2fe0ba32:34999 {}] master.HMaster(1795): Stopping service threads
2024-11-24T04:32:46,478 INFO  [M:0;fadb2fe0ba32:34999 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-24T04:32:46,478 INFO  [M:0;fadb2fe0ba32:34999 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-24T04:32:46,478 INFO  [M:0;fadb2fe0ba32:34999 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-24T04:32:46,478 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-24T04:32:46,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34999-0x101483e98110000, quorum=127.0.0.1:64360, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-24T04:32:46,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34999-0x101483e98110000, quorum=127.0.0.1:64360, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:32:46,479 DEBUG [M:0;fadb2fe0ba32:34999 {}] zookeeper.ZKUtil(347): master:34999-0x101483e98110000, quorum=127.0.0.1:64360, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-24T04:32:46,479 WARN  [M:0;fadb2fe0ba32:34999 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-24T04:32:46,480 INFO  [M:0;fadb2fe0ba32:34999 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/.lastflushedseqids
2024-11-24T04:32:46,484 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741877_1053 (size=228)
2024-11-24T04:32:46,484 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36545 is added to blk_1073741877_1053 (size=228)
2024-11-24T04:32:46,485 INFO  [M:0;fadb2fe0ba32:34999 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-24T04:32:46,485 INFO  [M:0;fadb2fe0ba32:34999 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-24T04:32:46,485 DEBUG [M:0;fadb2fe0ba32:34999 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-24T04:32:46,485 INFO  [M:0;fadb2fe0ba32:34999 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T04:32:46,485 DEBUG [M:0;fadb2fe0ba32:34999 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T04:32:46,485 DEBUG [M:0;fadb2fe0ba32:34999 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-24T04:32:46,485 DEBUG [M:0;fadb2fe0ba32:34999 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T04:32:46,485 INFO  [M:0;fadb2fe0ba32:34999 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=51.43 KB heapSize=63.36 KB
2024-11-24T04:32:46,501 DEBUG [M:0;fadb2fe0ba32:34999 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/93d86ca349074c07a591fa010038767d is 82, key is hbase:meta,,1/info:regioninfo/1732422709462/Put/seqid=0
2024-11-24T04:32:46,505 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741878_1054 (size=5672)
2024-11-24T04:32:46,505 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36545 is added to blk_1073741878_1054 (size=5672)
2024-11-24T04:32:46,505 INFO  [M:0;fadb2fe0ba32:34999 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/93d86ca349074c07a591fa010038767d
2024-11-24T04:32:46,524 DEBUG [M:0;fadb2fe0ba32:34999 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/95a0c0120d564216943149e9f3f6d453 is 751, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732422709956/Put/seqid=0
2024-11-24T04:32:46,528 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741879_1055 (size=7091)
2024-11-24T04:32:46,528 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36545 is added to blk_1073741879_1055 (size=7091)
2024-11-24T04:32:46,529 INFO  [M:0;fadb2fe0ba32:34999 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=50.83 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/95a0c0120d564216943149e9f3f6d453
2024-11-24T04:32:46,533 INFO  [M:0;fadb2fe0ba32:34999 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 95a0c0120d564216943149e9f3f6d453
2024-11-24T04:32:46,547 DEBUG [M:0;fadb2fe0ba32:34999 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/54aad23c38e54b43a86c42f91c9d4cb3 is 69, key is fadb2fe0ba32,33299,1732422708667/rs:state/1732422708902/Put/seqid=0
2024-11-24T04:32:46,551 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741880_1056 (size=5156)
2024-11-24T04:32:46,552 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36545 is added to blk_1073741880_1056 (size=5156)
2024-11-24T04:32:46,552 INFO  [M:0;fadb2fe0ba32:34999 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/54aad23c38e54b43a86c42f91c9d4cb3
2024-11-24T04:32:46,570 DEBUG [M:0;fadb2fe0ba32:34999 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/120e8b3077144c2ba8371ed5eb1243b1 is 52, key is load_balancer_on/state:d/1732422709591/Put/seqid=0
2024-11-24T04:32:46,574 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36545 is added to blk_1073741881_1057 (size=5056)
2024-11-24T04:32:46,574 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33299-0x101483e98110001, quorum=127.0.0.1:64360, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-24T04:32:46,574 INFO  [RS:0;fadb2fe0ba32:33299 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-24T04:32:46,574 INFO  [RS:0;fadb2fe0ba32:33299 {}] regionserver.HRegionServer(1031): Exiting; stopping=fadb2fe0ba32,33299,1732422708667; zookeeper connection closed.
2024-11-24T04:32:46,574 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33299-0x101483e98110001, quorum=127.0.0.1:64360, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-24T04:32:46,574 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741881_1057 (size=5056)
2024-11-24T04:32:46,574 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3fc66fc2 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3fc66fc2
2024-11-24T04:32:46,575 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-24T04:32:46,575 INFO  [M:0;fadb2fe0ba32:34999 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/120e8b3077144c2ba8371ed5eb1243b1
2024-11-24T04:32:46,579 DEBUG [M:0;fadb2fe0ba32:34999 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/93d86ca349074c07a591fa010038767d as hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/93d86ca349074c07a591fa010038767d
2024-11-24T04:32:46,583 INFO  [M:0;fadb2fe0ba32:34999 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/93d86ca349074c07a591fa010038767d, entries=8, sequenceid=125, filesize=5.5 K
2024-11-24T04:32:46,584 DEBUG [M:0;fadb2fe0ba32:34999 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/95a0c0120d564216943149e9f3f6d453 as hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/95a0c0120d564216943149e9f3f6d453
2024-11-24T04:32:46,588 INFO  [M:0;fadb2fe0ba32:34999 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 95a0c0120d564216943149e9f3f6d453
2024-11-24T04:32:46,588 INFO  [M:0;fadb2fe0ba32:34999 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/95a0c0120d564216943149e9f3f6d453, entries=13, sequenceid=125, filesize=6.9 K
2024-11-24T04:32:46,589 DEBUG [M:0;fadb2fe0ba32:34999 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/54aad23c38e54b43a86c42f91c9d4cb3 as hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/54aad23c38e54b43a86c42f91c9d4cb3
2024-11-24T04:32:46,593 INFO  [M:0;fadb2fe0ba32:34999 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/54aad23c38e54b43a86c42f91c9d4cb3, entries=1, sequenceid=125, filesize=5.0 K
2024-11-24T04:32:46,594 DEBUG [M:0;fadb2fe0ba32:34999 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/120e8b3077144c2ba8371ed5eb1243b1 as hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/120e8b3077144c2ba8371ed5eb1243b1
2024-11-24T04:32:46,597 INFO  [M:0;fadb2fe0ba32:34999 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36181/user/jenkins/test-data/9ca16472-3144-bffe-210d-0197cfa88fa6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/120e8b3077144c2ba8371ed5eb1243b1, entries=1, sequenceid=125, filesize=4.9 K
2024-11-24T04:32:46,598 INFO  [M:0;fadb2fe0ba32:34999 {}] regionserver.HRegion(3140): Finished flush of dataSize ~51.43 KB/52663, heapSize ~63.30 KB/64816, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 113ms, sequenceid=125, compaction requested=false
2024-11-24T04:32:46,599 INFO  [M:0;fadb2fe0ba32:34999 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T04:32:46,600 DEBUG [M:0;fadb2fe0ba32:34999 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732422766485Disabling compacts and flushes for region at 1732422766485Disabling writes for close at 1732422766485Obtaining lock to block concurrent updates at 1732422766485Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732422766485Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=52663, getHeapSize=64816, getOffHeapSize=0, getCellsCount=148 at 1732422766485Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732422766486 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732422766486Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732422766501 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732422766501Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732422766509 (+8 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732422766524 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732422766524Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732422766533 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732422766547 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732422766547Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732422766556 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732422766569 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732422766569Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@559fbaf9: reopening flushed file at 1732422766579 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6d6fb012: reopening flushed file at 1732422766583 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@59ac08c2: reopening flushed file at 1732422766588 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6cfdbe0a: reopening flushed file at 1732422766593 (+5 ms)Finished flush of dataSize ~51.43 KB/52663, heapSize ~63.30 KB/64816, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 113ms, sequenceid=125, compaction requested=false at 1732422766598 (+5 ms)Writing region close event to WAL at 1732422766599 (+1 ms)Closed at 1732422766599
2024-11-24T04:32:46,600 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:32:46,600 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:32:46,600 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:32:46,600 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:32:46,600 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:32:46,602 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36479 is added to blk_1073741830_1006 (size=61332)
2024-11-24T04:32:46,602 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36545 is added to blk_1073741830_1006 (size=61332)
2024-11-24T04:32:46,603 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-24T04:32:46,603 INFO  [M:0;fadb2fe0ba32:34999 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-24T04:32:46,603 INFO  [M:0;fadb2fe0ba32:34999 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34999
2024-11-24T04:32:46,603 INFO  [M:0;fadb2fe0ba32:34999 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-24T04:32:46,705 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34999-0x101483e98110000, quorum=127.0.0.1:64360, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-24T04:32:46,705 INFO  [M:0;fadb2fe0ba32:34999 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-24T04:32:46,705 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34999-0x101483e98110000, quorum=127.0.0.1:64360, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-24T04:32:46,709 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@76385d85{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T04:32:46,709 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@284a8a01{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-24T04:32:46,710 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-24T04:32:46,710 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7193a060{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-24T04:32:46,710 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@444d0b71{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87b7a640-4933-f8f1-d302-11da82f8fd82/hadoop.log.dir/,STOPPED}
2024-11-24T04:32:46,712 WARN  [BP-873427653-172.17.0.2-1732422707861 heartbeating to localhost/127.0.0.1:36181 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-24T04:32:46,712 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-24T04:32:46,712 WARN  [BP-873427653-172.17.0.2-1732422707861 heartbeating to localhost/127.0.0.1:36181 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-873427653-172.17.0.2-1732422707861 (Datanode Uuid 883abd11-f372-4510-952d-7902c08af9d6) service to localhost/127.0.0.1:36181
2024-11-24T04:32:46,712 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-24T04:32:46,712 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87b7a640-4933-f8f1-d302-11da82f8fd82/cluster_a83580c7-cecb-4f73-b131-04d66d148ce4/data/data3/current/BP-873427653-172.17.0.2-1732422707861 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T04:32:46,713 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87b7a640-4933-f8f1-d302-11da82f8fd82/cluster_a83580c7-cecb-4f73-b131-04d66d148ce4/data/data4/current/BP-873427653-172.17.0.2-1732422707861 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T04:32:46,713 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-24T04:32:46,715 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@27e7702e{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T04:32:46,715 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@33382c80{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-24T04:32:46,715 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-24T04:32:46,715 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3c10c155{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-24T04:32:46,715 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4a83b9f6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87b7a640-4933-f8f1-d302-11da82f8fd82/hadoop.log.dir/,STOPPED}
2024-11-24T04:32:46,717 WARN  [BP-873427653-172.17.0.2-1732422707861 heartbeating to localhost/127.0.0.1:36181 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-24T04:32:46,717 WARN  [BP-873427653-172.17.0.2-1732422707861 heartbeating to localhost/127.0.0.1:36181 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-873427653-172.17.0.2-1732422707861 (Datanode Uuid 6829c1c6-4bf3-412b-a59c-0bd2436c658b) service to localhost/127.0.0.1:36181
2024-11-24T04:32:46,717 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-24T04:32:46,717 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-24T04:32:46,717 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87b7a640-4933-f8f1-d302-11da82f8fd82/cluster_a83580c7-cecb-4f73-b131-04d66d148ce4/data/data1/current/BP-873427653-172.17.0.2-1732422707861 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T04:32:46,717 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87b7a640-4933-f8f1-d302-11da82f8fd82/cluster_a83580c7-cecb-4f73-b131-04d66d148ce4/data/data2/current/BP-873427653-172.17.0.2-1732422707861 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T04:32:46,717 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-24T04:32:46,724 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2ffc76d1{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-24T04:32:46,724 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@640c9e09{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-24T04:32:46,724 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-24T04:32:46,724 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7e466bdb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-24T04:32:46,724 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3d320e1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87b7a640-4933-f8f1-d302-11da82f8fd82/hadoop.log.dir/,STOPPED}
2024-11-24T04:32:46,732 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-24T04:32:46,767 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-24T04:32:46,777 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRolling Thread=230 (was 205)
Potentially hanging thread: HMaster-EventLoopGroup-14-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-41-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-41-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-39-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-38-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-14-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-38-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:36181 from jenkins.hfs.6
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36181
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-40-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-39-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-41-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-38-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36181
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-40-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-40-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36181
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-14-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:36181
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:36181 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.6@localhost:36181
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:36181 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-39-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=503 (was 483) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=16 (was 40), ProcessCount=11 (was 11), AvailableMemoryMB=6403 (was 6464)
2024-11-24T04:32:46,785 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=230, OpenFileDescriptor=503, MaxFileDescriptor=1048576, SystemLoadAverage=16, ProcessCount=11, AvailableMemoryMB=6403
2024-11-24T04:32:46,785 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-24T04:32:46,785 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87b7a640-4933-f8f1-d302-11da82f8fd82/hadoop.log.dir so I do NOT create it in target/test-data/239376e9-b709-06b1-2009-9af67614e40c
2024-11-24T04:32:46,785 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/87b7a640-4933-f8f1-d302-11da82f8fd82/hadoop.tmp.dir so I do NOT create it in target/test-data/239376e9-b709-06b1-2009-9af67614e40c
2024-11-24T04:32:46,785 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/239376e9-b709-06b1-2009-9af67614e40c/cluster_ba9c276b-166c-9d23-ed5c-f05f16006543, deleteOnExit=true
2024-11-24T04:32:46,785 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-24T04:32:46,785 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/239376e9-b709-06b1-2009-9af67614e40c/test.cache.data in system properties and HBase conf
2024-11-24T04:32:46,785 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/239376e9-b709-06b1-2009-9af67614e40c/hadoop.tmp.dir in system properties and HBase conf
2024-11-24T04:32:46,785 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/239376e9-b709-06b1-2009-9af67614e40c/hadoop.log.dir in system properties and HBase conf
2024-11-24T04:32:46,785 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/239376e9-b709-06b1-2009-9af67614e40c/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-24T04:32:46,785 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/239376e9-b709-06b1-2009-9af67614e40c/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-24T04:32:46,785 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-24T04:32:46,785 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-24T04:32:46,786 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/239376e9-b709-06b1-2009-9af67614e40c/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-24T04:32:46,786 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/239376e9-b709-06b1-2009-9af67614e40c/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-24T04:32:46,786 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/239376e9-b709-06b1-2009-9af67614e40c/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-24T04:32:46,786 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/239376e9-b709-06b1-2009-9af67614e40c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-24T04:32:46,786 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/239376e9-b709-06b1-2009-9af67614e40c/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-24T04:32:46,786 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/239376e9-b709-06b1-2009-9af67614e40c/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-24T04:32:46,786 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/239376e9-b709-06b1-2009-9af67614e40c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-24T04:32:46,786 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/239376e9-b709-06b1-2009-9af67614e40c/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-24T04:32:46,786 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/239376e9-b709-06b1-2009-9af67614e40c/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-24T04:32:46,786 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/239376e9-b709-06b1-2009-9af67614e40c/nfs.dump.dir in system properties and HBase conf
2024-11-24T04:32:46,786 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/239376e9-b709-06b1-2009-9af67614e40c/java.io.tmpdir in system properties and HBase conf
2024-11-24T04:32:46,786 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/239376e9-b709-06b1-2009-9af67614e40c/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-24T04:32:46,786 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/239376e9-b709-06b1-2009-9af67614e40c/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-24T04:32:46,786 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/239376e9-b709-06b1-2009-9af67614e40c/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-24T04:32:46,799 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-24T04:32:46,863 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T04:32:46,866 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-24T04:32:46,872 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-24T04:32:46,873 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-24T04:32:46,873 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-24T04:32:46,873 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T04:32:46,874 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@63ebf311{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/239376e9-b709-06b1-2009-9af67614e40c/hadoop.log.dir/,AVAILABLE}
2024-11-24T04:32:46,874 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:46,874 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:46,874 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@64825638{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-24T04:32:46,916 INFO  [regionserver/fadb2fe0ba32:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-24T04:32:46,987 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2a7c9e07{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/239376e9-b709-06b1-2009-9af67614e40c/java.io.tmpdir/jetty-localhost-42143-hadoop-hdfs-3_4_1-tests_jar-_-any-9397747588036145188/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-24T04:32:46,988 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@225265b7{HTTP/1.1, (http/1.1)}{localhost:42143}
2024-11-24T04:32:46,988 INFO  [Time-limited test {}] server.Server(415): Started @296049ms
2024-11-24T04:32:47,001 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-24T04:32:47,050 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T04:32:47,053 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-24T04:32:47,054 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-24T04:32:47,054 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-24T04:32:47,054 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-24T04:32:47,054 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@142dd349{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/239376e9-b709-06b1-2009-9af67614e40c/hadoop.log.dir/,AVAILABLE}
2024-11-24T04:32:47,055 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1802ed83{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-24T04:32:47,169 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@d6e133d{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/239376e9-b709-06b1-2009-9af67614e40c/java.io.tmpdir/jetty-localhost-46245-hadoop-hdfs-3_4_1-tests_jar-_-any-1910448994820931880/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T04:32:47,169 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@652ac650{HTTP/1.1, (http/1.1)}{localhost:46245}
2024-11-24T04:32:47,169 INFO  [Time-limited test {}] server.Server(415): Started @296231ms
2024-11-24T04:32:47,171 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-24T04:32:47,200 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-24T04:32:47,202 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-24T04:32:47,203 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-24T04:32:47,203 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-24T04:32:47,203 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-24T04:32:47,204 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@677f535e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/239376e9-b709-06b1-2009-9af67614e40c/hadoop.log.dir/,AVAILABLE}
2024-11-24T04:32:47,204 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@35a6030c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-24T04:32:47,290 WARN  [Thread-2466 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/239376e9-b709-06b1-2009-9af67614e40c/cluster_ba9c276b-166c-9d23-ed5c-f05f16006543/data/data1/current/BP-1321689454-172.17.0.2-1732422766805/current, will proceed with Du for space computation calculation, 
2024-11-24T04:32:47,290 WARN  [Thread-2467 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/239376e9-b709-06b1-2009-9af67614e40c/cluster_ba9c276b-166c-9d23-ed5c-f05f16006543/data/data2/current/BP-1321689454-172.17.0.2-1732422766805/current, will proceed with Du for space computation calculation, 
2024-11-24T04:32:47,306 WARN  [Thread-2445 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-24T04:32:47,309 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x740187c5093517e4 with lease ID 0x379457ca53b85eb2: Processing first storage report for DS-48074329-1723-4c1b-9194-623e0b558007 from datanode DatanodeRegistration(127.0.0.1:38705, datanodeUuid=9d57edc0-9bb2-4f98-bae5-0b3df1e6dd01, infoPort=43777, infoSecurePort=0, ipcPort=37889, storageInfo=lv=-57;cid=testClusterID;nsid=153765504;c=1732422766805)
2024-11-24T04:32:47,309 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x740187c5093517e4 with lease ID 0x379457ca53b85eb2: from storage DS-48074329-1723-4c1b-9194-623e0b558007 node DatanodeRegistration(127.0.0.1:38705, datanodeUuid=9d57edc0-9bb2-4f98-bae5-0b3df1e6dd01, infoPort=43777, infoSecurePort=0, ipcPort=37889, storageInfo=lv=-57;cid=testClusterID;nsid=153765504;c=1732422766805), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T04:32:47,309 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x740187c5093517e4 with lease ID 0x379457ca53b85eb2: Processing first storage report for DS-eb9ff189-9741-4eee-b416-ffc7e1ad0e32 from datanode DatanodeRegistration(127.0.0.1:38705, datanodeUuid=9d57edc0-9bb2-4f98-bae5-0b3df1e6dd01, infoPort=43777, infoSecurePort=0, ipcPort=37889, storageInfo=lv=-57;cid=testClusterID;nsid=153765504;c=1732422766805)
2024-11-24T04:32:47,309 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x740187c5093517e4 with lease ID 0x379457ca53b85eb2: from storage DS-eb9ff189-9741-4eee-b416-ffc7e1ad0e32 node DatanodeRegistration(127.0.0.1:38705, datanodeUuid=9d57edc0-9bb2-4f98-bae5-0b3df1e6dd01, infoPort=43777, infoSecurePort=0, ipcPort=37889, storageInfo=lv=-57;cid=testClusterID;nsid=153765504;c=1732422766805), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T04:32:47,332 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3698cd86{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/239376e9-b709-06b1-2009-9af67614e40c/java.io.tmpdir/jetty-localhost-37841-hadoop-hdfs-3_4_1-tests_jar-_-any-15602737354951301824/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T04:32:47,333 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@418bce8c{HTTP/1.1, (http/1.1)}{localhost:37841}
2024-11-24T04:32:47,333 INFO  [Time-limited test {}] server.Server(415): Started @296394ms
2024-11-24T04:32:47,334 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-24T04:32:47,426 WARN  [Thread-2492 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/239376e9-b709-06b1-2009-9af67614e40c/cluster_ba9c276b-166c-9d23-ed5c-f05f16006543/data/data3/current/BP-1321689454-172.17.0.2-1732422766805/current, will proceed with Du for space computation calculation, 
2024-11-24T04:32:47,426 WARN  [Thread-2493 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/239376e9-b709-06b1-2009-9af67614e40c/cluster_ba9c276b-166c-9d23-ed5c-f05f16006543/data/data4/current/BP-1321689454-172.17.0.2-1732422766805/current, will proceed with Du for space computation calculation, 
2024-11-24T04:32:47,448 WARN  [Thread-2481 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-24T04:32:47,450 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x295c0d37c541544c with lease ID 0x379457ca53b85eb3: Processing first storage report for DS-2853df03-39c0-4a32-900a-71cdebea0677 from datanode DatanodeRegistration(127.0.0.1:40251, datanodeUuid=8683ebfa-83a3-4661-af86-01fabb3439d6, infoPort=45377, infoSecurePort=0, ipcPort=42625, storageInfo=lv=-57;cid=testClusterID;nsid=153765504;c=1732422766805)
2024-11-24T04:32:47,450 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x295c0d37c541544c with lease ID 0x379457ca53b85eb3: from storage DS-2853df03-39c0-4a32-900a-71cdebea0677 node DatanodeRegistration(127.0.0.1:40251, datanodeUuid=8683ebfa-83a3-4661-af86-01fabb3439d6, infoPort=45377, infoSecurePort=0, ipcPort=42625, storageInfo=lv=-57;cid=testClusterID;nsid=153765504;c=1732422766805), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T04:32:47,450 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x295c0d37c541544c with lease ID 0x379457ca53b85eb3: Processing first storage report for DS-e5aa91c0-7a15-4bf4-8cb8-0c0f984bbe12 from datanode DatanodeRegistration(127.0.0.1:40251, datanodeUuid=8683ebfa-83a3-4661-af86-01fabb3439d6, infoPort=45377, infoSecurePort=0, ipcPort=42625, storageInfo=lv=-57;cid=testClusterID;nsid=153765504;c=1732422766805)
2024-11-24T04:32:47,450 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x295c0d37c541544c with lease ID 0x379457ca53b85eb3: from storage DS-e5aa91c0-7a15-4bf4-8cb8-0c0f984bbe12 node DatanodeRegistration(127.0.0.1:40251, datanodeUuid=8683ebfa-83a3-4661-af86-01fabb3439d6, infoPort=45377, infoSecurePort=0, ipcPort=42625, storageInfo=lv=-57;cid=testClusterID;nsid=153765504;c=1732422766805), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-24T04:32:47,454 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/239376e9-b709-06b1-2009-9af67614e40c
2024-11-24T04:32:47,457 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/239376e9-b709-06b1-2009-9af67614e40c/cluster_ba9c276b-166c-9d23-ed5c-f05f16006543/zookeeper_0, clientPort=56125, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/239376e9-b709-06b1-2009-9af67614e40c/cluster_ba9c276b-166c-9d23-ed5c-f05f16006543/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/239376e9-b709-06b1-2009-9af67614e40c/cluster_ba9c276b-166c-9d23-ed5c-f05f16006543/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-24T04:32:47,457 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=56125
2024-11-24T04:32:47,458 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T04:32:47,459 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T04:32:47,466 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38705 is added to blk_1073741825_1001 (size=7)
2024-11-24T04:32:47,467 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40251 is added to blk_1073741825_1001 (size=7)
2024-11-24T04:32:47,467 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:37089/user/jenkins/test-data/6646e03d-3e80-85d7-fac8-e27673f4f898 with version=8
2024-11-24T04:32:47,467 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:35431/user/jenkins/test-data/a947dfed-d761-5e4b-1680-018036a8a165/hbase-staging
2024-11-24T04:32:47,469 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/fadb2fe0ba32:0 server-side Connection retries=45
2024-11-24T04:32:47,469 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-24T04:32:47,470 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-24T04:32:47,470 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-24T04:32:47,470 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-24T04:32:47,470 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-24T04:32:47,470 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-24T04:32:47,470 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-24T04:32:47,470 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45215
2024-11-24T04:32:47,471 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:45215 connecting to ZooKeeper ensemble=127.0.0.1:56125
2024-11-24T04:32:47,477 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:452150x0, quorum=127.0.0.1:56125, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-24T04:32:47,477 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:45215-0x101483f7df00000 connected
2024-11-24T04:32:47,492 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T04:32:47,493 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T04:32:47,495 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45215-0x101483f7df00000, quorum=127.0.0.1:56125, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-24T04:32:47,495 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:37089/user/jenkins/test-data/6646e03d-3e80-85d7-fac8-e27673f4f898, hbase.cluster.distributed=false
2024-11-24T04:32:47,497 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45215-0x101483f7df00000, quorum=127.0.0.1:56125, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-24T04:32:47,497 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45215
2024-11-24T04:32:47,497 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45215
2024-11-24T04:32:47,497 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45215
2024-11-24T04:32:47,498 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45215
2024-11-24T04:32:47,498 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45215
2024-11-24T04:32:47,513 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/fadb2fe0ba32:0 server-side Connection retries=45
2024-11-24T04:32:47,513 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-24T04:32:47,513 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-24T04:32:47,513 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-24T04:32:47,513 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-24T04:32:47,513 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-24T04:32:47,513 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-24T04:32:47,513 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-24T04:32:47,514 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33431
2024-11-24T04:32:47,515 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33431 connecting to ZooKeeper ensemble=127.0.0.1:56125
2024-11-24T04:32:47,515 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T04:32:47,517 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T04:32:47,520 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:334310x0, quorum=127.0.0.1:56125, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-24T04:32:47,521 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33431-0x101483f7df00001, quorum=127.0.0.1:56125, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-24T04:32:47,521 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33431-0x101483f7df00001 connected
2024-11-24T04:32:47,521 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-24T04:32:47,522 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-24T04:32:47,522 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33431-0x101483f7df00001, quorum=127.0.0.1:56125, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-24T04:32:47,523 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33431-0x101483f7df00001, quorum=127.0.0.1:56125, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-24T04:32:47,523 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33431
2024-11-24T04:32:47,524 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33431
2024-11-24T04:32:47,524 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33431
2024-11-24T04:32:47,524 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33431
2024-11-24T04:32:47,525 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33431
2024-11-24T04:32:47,536 DEBUG [M:0;fadb2fe0ba32:45215 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;fadb2fe0ba32:45215
2024-11-24T04:32:47,536 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/fadb2fe0ba32,45215,1732422767469
2024-11-24T04:32:47,541 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33431-0x101483f7df00001, quorum=127.0.0.1:56125, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-24T04:32:47,541 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45215-0x101483f7df00000, quorum=127.0.0.1:56125, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-24T04:32:47,541 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45215-0x101483f7df00000, quorum=127.0.0.1:56125, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/fadb2fe0ba32,45215,1732422767469
2024-11-24T04:32:47,542 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33431-0x101483f7df00001, quorum=127.0.0.1:56125, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-24T04:32:47,542 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45215-0x101483f7df00000, quorum=127.0.0.1:56125, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:32:47,542 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33431-0x101483f7df00001, quorum=127.0.0.1:56125, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:32:47,543 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45215-0x101483f7df00000, quorum=127.0.0.1:56125, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-24T04:32:47,543 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/fadb2fe0ba32,45215,1732422767469 from backup master directory
2024-11-24T04:32:47,544 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45215-0x101483f7df00000, quorum=127.0.0.1:56125, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/fadb2fe0ba32,45215,1732422767469
2024-11-24T04:32:47,544 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33431-0x101483f7df00001, quorum=127.0.0.1:56125, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-24T04:32:47,544 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45215-0x101483f7df00000, quorum=127.0.0.1:56125, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-24T04:32:47,544 WARN  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-24T04:32:47,544 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=fadb2fe0ba32,45215,1732422767469
2024-11-24T04:32:47,548 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:37089/user/jenkins/test-data/6646e03d-3e80-85d7-fac8-e27673f4f898/hbase.id] with ID: fe5b4538-2f36-43f4-993b-f270d4709ca1
2024-11-24T04:32:47,548 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:37089/user/jenkins/test-data/6646e03d-3e80-85d7-fac8-e27673f4f898/.tmp/hbase.id
2024-11-24T04:32:47,553 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38705 is added to blk_1073741826_1002 (size=42)
2024-11-24T04:32:47,553 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40251 is added to blk_1073741826_1002 (size=42)
2024-11-24T04:32:47,554 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:37089/user/jenkins/test-data/6646e03d-3e80-85d7-fac8-e27673f4f898/.tmp/hbase.id]:[hdfs://localhost:37089/user/jenkins/test-data/6646e03d-3e80-85d7-fac8-e27673f4f898/hbase.id]
2024-11-24T04:32:47,563 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T04:32:47,563 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-24T04:32:47,564 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-11-24T04:32:47,566 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33431-0x101483f7df00001, quorum=127.0.0.1:56125, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:32:47,566 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45215-0x101483f7df00000, quorum=127.0.0.1:56125, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:32:47,574 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40251 is added to blk_1073741827_1003 (size=196)
2024-11-24T04:32:47,574 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38705 is added to blk_1073741827_1003 (size=196)
2024-11-24T04:32:47,575 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-24T04:32:47,575 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-24T04:32:47,576 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-24T04:32:47,581 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40251 is added to blk_1073741828_1004 (size=1189)
2024-11-24T04:32:47,582 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38705 is added to blk_1073741828_1004 (size=1189)
2024-11-24T04:32:47,582 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:37089/user/jenkins/test-data/6646e03d-3e80-85d7-fac8-e27673f4f898/MasterData/data/master/store
2024-11-24T04:32:47,587 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38705 is added to blk_1073741829_1005 (size=34)
2024-11-24T04:32:47,587 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40251 is added to blk_1073741829_1005 (size=34)
2024-11-24T04:32:47,588 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T04:32:47,588 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-24T04:32:47,588 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T04:32:47,588 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T04:32:47,588 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-24T04:32:47,588 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T04:32:47,588 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T04:32:47,588 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732422767588Disabling compacts and flushes for region at 1732422767588Disabling writes for close at 1732422767588Writing region close event to WAL at 1732422767588Closed at 1732422767588
2024-11-24T04:32:47,589 WARN  [master/fadb2fe0ba32:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:37089/user/jenkins/test-data/6646e03d-3e80-85d7-fac8-e27673f4f898/MasterData/data/master/store/.initializing
2024-11-24T04:32:47,589 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:37089/user/jenkins/test-data/6646e03d-3e80-85d7-fac8-e27673f4f898/MasterData/WALs/fadb2fe0ba32,45215,1732422767469
2024-11-24T04:32:47,591 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fadb2fe0ba32%2C45215%2C1732422767469, suffix=, logDir=hdfs://localhost:37089/user/jenkins/test-data/6646e03d-3e80-85d7-fac8-e27673f4f898/MasterData/WALs/fadb2fe0ba32,45215,1732422767469, archiveDir=hdfs://localhost:37089/user/jenkins/test-data/6646e03d-3e80-85d7-fac8-e27673f4f898/MasterData/oldWALs, maxLogs=10
2024-11-24T04:32:47,591 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor fadb2fe0ba32%2C45215%2C1732422767469.1732422767591
2024-11-24T04:32:47,595 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6646e03d-3e80-85d7-fac8-e27673f4f898/MasterData/WALs/fadb2fe0ba32,45215,1732422767469/fadb2fe0ba32%2C45215%2C1732422767469.1732422767591
2024-11-24T04:32:47,597 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45377:45377),(127.0.0.1/127.0.0.1:43777:43777)]
2024-11-24T04:32:47,602 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-24T04:32:47,602 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T04:32:47,602 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T04:32:47,602 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T04:32:47,603 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-24T04:32:47,604 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-24T04:32:47,604 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:32:47,605 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T04:32:47,605 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-24T04:32:47,606 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-24T04:32:47,606 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:32:47,606 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-24T04:32:47,606 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-24T04:32:47,607 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-24T04:32:47,607 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:32:47,607 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-24T04:32:47,607 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-24T04:32:47,608 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-24T04:32:47,608 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:32:47,608 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-24T04:32:47,609 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T04:32:47,609 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37089/user/jenkins/test-data/6646e03d-3e80-85d7-fac8-e27673f4f898/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-24T04:32:47,609 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37089/user/jenkins/test-data/6646e03d-3e80-85d7-fac8-e27673f4f898/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-24T04:32:47,610 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T04:32:47,610 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T04:32:47,611 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-24T04:32:47,612 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-24T04:32:47,615 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37089/user/jenkins/test-data/6646e03d-3e80-85d7-fac8-e27673f4f898/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-24T04:32:47,615 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=796350, jitterRate=0.012611404061317444}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-24T04:32:47,616 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1732422767602Initializing all the Stores at 1732422767603 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732422767603Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422767603Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422767603Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422767603Cleaning up temporary data from old regions at 1732422767610 (+7 ms)Region opened successfully at 1732422767616 (+6 ms)
2024-11-24T04:32:47,616 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-24T04:32:47,618 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3eab62e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fadb2fe0ba32/172.17.0.2:0
2024-11-24T04:32:47,619 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-24T04:32:47,619 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-24T04:32:47,619 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-24T04:32:47,619 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-24T04:32:47,620 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-24T04:32:47,620 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-24T04:32:47,620 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-24T04:32:47,622 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-24T04:32:47,623 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45215-0x101483f7df00000, quorum=127.0.0.1:56125, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-24T04:32:47,625 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-24T04:32:47,626 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-24T04:32:47,626 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45215-0x101483f7df00000, quorum=127.0.0.1:56125, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-24T04:32:47,627 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-24T04:32:47,627 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-24T04:32:47,628 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45215-0x101483f7df00000, quorum=127.0.0.1:56125, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-24T04:32:47,629 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-24T04:32:47,630 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45215-0x101483f7df00000, quorum=127.0.0.1:56125, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-24T04:32:47,631 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-24T04:32:47,632 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45215-0x101483f7df00000, quorum=127.0.0.1:56125, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-24T04:32:47,635 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-24T04:32:47,636 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33431-0x101483f7df00001, quorum=127.0.0.1:56125, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-24T04:32:47,636 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45215-0x101483f7df00000, quorum=127.0.0.1:56125, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-24T04:32:47,637 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33431-0x101483f7df00001, quorum=127.0.0.1:56125, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:32:47,637 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45215-0x101483f7df00000, quorum=127.0.0.1:56125, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:32:47,637 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=fadb2fe0ba32,45215,1732422767469, sessionid=0x101483f7df00000, setting cluster-up flag (Was=false)
2024-11-24T04:32:47,639 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33431-0x101483f7df00001, quorum=127.0.0.1:56125, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:32:47,639 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45215-0x101483f7df00000, quorum=127.0.0.1:56125, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:32:47,645 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-24T04:32:47,646 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=fadb2fe0ba32,45215,1732422767469
2024-11-24T04:32:47,649 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45215-0x101483f7df00000, quorum=127.0.0.1:56125, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:32:47,649 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33431-0x101483f7df00001, quorum=127.0.0.1:56125, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:32:47,653 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-24T04:32:47,654 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=fadb2fe0ba32,45215,1732422767469
2024-11-24T04:32:47,655 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:37089/user/jenkins/test-data/6646e03d-3e80-85d7-fac8-e27673f4f898/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-24T04:32:47,656 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-24T04:32:47,657 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-24T04:32:47,657 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-24T04:32:47,657 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: fadb2fe0ba32,45215,1732422767469
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-24T04:32:47,658 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/fadb2fe0ba32:0, corePoolSize=5, maxPoolSize=5
2024-11-24T04:32:47,658 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/fadb2fe0ba32:0, corePoolSize=5, maxPoolSize=5
2024-11-24T04:32:47,658 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/fadb2fe0ba32:0, corePoolSize=5, maxPoolSize=5
2024-11-24T04:32:47,658 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/fadb2fe0ba32:0, corePoolSize=5, maxPoolSize=5
2024-11-24T04:32:47,658 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/fadb2fe0ba32:0, corePoolSize=10, maxPoolSize=10
2024-11-24T04:32:47,658 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:32:47,658 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/fadb2fe0ba32:0, corePoolSize=2, maxPoolSize=2
2024-11-24T04:32:47,658 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:32:47,659 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732422797659
2024-11-24T04:32:47,659 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-24T04:32:47,659 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-24T04:32:47,659 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-24T04:32:47,659 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-24T04:32:47,659 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-24T04:32:47,659 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-24T04:32:47,659 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-24T04:32:47,659 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-24T04:32:47,659 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-24T04:32:47,660 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-24T04:32:47,660 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-24T04:32:47,660 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-24T04:32:47,660 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-24T04:32:47,660 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-24T04:32:47,660 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:32:47,660 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/fadb2fe0ba32:0:becomeActiveMaster-HFileCleaner.large.0-1732422767660,5,FailOnTimeoutGroup]
2024-11-24T04:32:47,660 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-24T04:32:47,661 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/fadb2fe0ba32:0:becomeActiveMaster-HFileCleaner.small.0-1732422767661,5,FailOnTimeoutGroup]
2024-11-24T04:32:47,661 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-24T04:32:47,661 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-24T04:32:47,661 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-24T04:32:47,661 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-24T04:32:47,666 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40251 is added to blk_1073741831_1007 (size=1321)
2024-11-24T04:32:47,666 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38705 is added to blk_1073741831_1007 (size=1321)
2024-11-24T04:32:47,666 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:37089/user/jenkins/test-data/6646e03d-3e80-85d7-fac8-e27673f4f898/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-24T04:32:47,667 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:37089/user/jenkins/test-data/6646e03d-3e80-85d7-fac8-e27673f4f898
2024-11-24T04:32:47,671 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40251 is added to blk_1073741832_1008 (size=32)
2024-11-24T04:32:47,671 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38705 is added to blk_1073741832_1008 (size=32)
2024-11-24T04:32:47,672 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T04:32:47,673 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-24T04:32:47,674 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-24T04:32:47,674 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:32:47,674 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T04:32:47,675 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-24T04:32:47,675 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-24T04:32:47,675 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:32:47,676 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T04:32:47,676 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-24T04:32:47,677 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-24T04:32:47,677 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:32:47,677 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T04:32:47,677 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-24T04:32:47,678 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-24T04:32:47,678 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:32:47,678 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T04:32:47,678 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-24T04:32:47,679 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37089/user/jenkins/test-data/6646e03d-3e80-85d7-fac8-e27673f4f898/data/hbase/meta/1588230740
2024-11-24T04:32:47,679 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37089/user/jenkins/test-data/6646e03d-3e80-85d7-fac8-e27673f4f898/data/hbase/meta/1588230740
2024-11-24T04:32:47,680 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-24T04:32:47,680 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-24T04:32:47,681 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-24T04:32:47,681 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-24T04:32:47,683 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37089/user/jenkins/test-data/6646e03d-3e80-85d7-fac8-e27673f4f898/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-24T04:32:47,683 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=782689, jitterRate=-0.004760578274726868}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-24T04:32:47,684 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1732422767672Initializing all the Stores at 1732422767673 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732422767673Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732422767673Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422767673Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732422767673Cleaning up temporary data from old regions at 1732422767680 (+7 ms)Region opened successfully at 1732422767684 (+4 ms)
2024-11-24T04:32:47,684 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-24T04:32:47,684 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-24T04:32:47,684 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-24T04:32:47,684 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-24T04:32:47,684 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-24T04:32:47,684 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-24T04:32:47,684 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732422767684Disabling compacts and flushes for region at 1732422767684Disabling writes for close at 1732422767684Writing region close event to WAL at 1732422767684Closed at 1732422767684
2024-11-24T04:32:47,685 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-24T04:32:47,685 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-24T04:32:47,686 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-24T04:32:47,686 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-24T04:32:47,687 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-24T04:32:47,726 INFO  [RS:0;fadb2fe0ba32:33431 {}] regionserver.HRegionServer(746): ClusterId : fe5b4538-2f36-43f4-993b-f270d4709ca1
2024-11-24T04:32:47,726 DEBUG [RS:0;fadb2fe0ba32:33431 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-24T04:32:47,729 DEBUG [RS:0;fadb2fe0ba32:33431 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-24T04:32:47,729 DEBUG [RS:0;fadb2fe0ba32:33431 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-24T04:32:47,730 DEBUG [RS:0;fadb2fe0ba32:33431 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-24T04:32:47,731 DEBUG [RS:0;fadb2fe0ba32:33431 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c506af0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fadb2fe0ba32/172.17.0.2:0
2024-11-24T04:32:47,743 DEBUG [RS:0;fadb2fe0ba32:33431 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;fadb2fe0ba32:33431
2024-11-24T04:32:47,743 INFO  [RS:0;fadb2fe0ba32:33431 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-24T04:32:47,743 INFO  [RS:0;fadb2fe0ba32:33431 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-24T04:32:47,743 DEBUG [RS:0;fadb2fe0ba32:33431 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-24T04:32:47,744 INFO  [RS:0;fadb2fe0ba32:33431 {}] regionserver.HRegionServer(2659): reportForDuty to master=fadb2fe0ba32,45215,1732422767469 with port=33431, startcode=1732422767512
2024-11-24T04:32:47,744 DEBUG [RS:0;fadb2fe0ba32:33431 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-24T04:32:47,746 INFO  [HMaster-EventLoopGroup-16-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57001, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.7 (auth:SIMPLE), service=RegionServerStatusService
2024-11-24T04:32:47,746 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45215 {}] master.ServerManager(363): Checking decommissioned status of RegionServer fadb2fe0ba32,33431,1732422767512
2024-11-24T04:32:47,747 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45215 {}] master.ServerManager(517): Registering regionserver=fadb2fe0ba32,33431,1732422767512
2024-11-24T04:32:47,748 DEBUG [RS:0;fadb2fe0ba32:33431 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:37089/user/jenkins/test-data/6646e03d-3e80-85d7-fac8-e27673f4f898
2024-11-24T04:32:47,748 DEBUG [RS:0;fadb2fe0ba32:33431 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:37089
2024-11-24T04:32:47,748 DEBUG [RS:0;fadb2fe0ba32:33431 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-24T04:32:47,749 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45215-0x101483f7df00000, quorum=127.0.0.1:56125, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-24T04:32:47,750 DEBUG [RS:0;fadb2fe0ba32:33431 {}] zookeeper.ZKUtil(111): regionserver:33431-0x101483f7df00001, quorum=127.0.0.1:56125, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/fadb2fe0ba32,33431,1732422767512
2024-11-24T04:32:47,750 WARN  [RS:0;fadb2fe0ba32:33431 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-24T04:32:47,750 INFO  [RS:0;fadb2fe0ba32:33431 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-24T04:32:47,750 DEBUG [RS:0;fadb2fe0ba32:33431 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:37089/user/jenkins/test-data/6646e03d-3e80-85d7-fac8-e27673f4f898/WALs/fadb2fe0ba32,33431,1732422767512
2024-11-24T04:32:47,750 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [fadb2fe0ba32,33431,1732422767512]
2024-11-24T04:32:47,753 INFO  [RS:0;fadb2fe0ba32:33431 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-24T04:32:47,757 INFO  [RS:0;fadb2fe0ba32:33431 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-24T04:32:47,757 INFO  [RS:0;fadb2fe0ba32:33431 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-24T04:32:47,757 INFO  [RS:0;fadb2fe0ba32:33431 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-24T04:32:47,758 INFO  [RS:0;fadb2fe0ba32:33431 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-24T04:32:47,758 INFO  [RS:0;fadb2fe0ba32:33431 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-24T04:32:47,758 INFO  [RS:0;fadb2fe0ba32:33431 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-24T04:32:47,758 DEBUG [RS:0;fadb2fe0ba32:33431 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:32:47,758 DEBUG [RS:0;fadb2fe0ba32:33431 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:32:47,758 DEBUG [RS:0;fadb2fe0ba32:33431 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:32:47,758 DEBUG [RS:0;fadb2fe0ba32:33431 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:32:47,758 DEBUG [RS:0;fadb2fe0ba32:33431 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:32:47,759 DEBUG [RS:0;fadb2fe0ba32:33431 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/fadb2fe0ba32:0, corePoolSize=2, maxPoolSize=2
2024-11-24T04:32:47,759 DEBUG [RS:0;fadb2fe0ba32:33431 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:32:47,759 DEBUG [RS:0;fadb2fe0ba32:33431 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:32:47,759 DEBUG [RS:0;fadb2fe0ba32:33431 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:32:47,759 DEBUG [RS:0;fadb2fe0ba32:33431 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:32:47,759 DEBUG [RS:0;fadb2fe0ba32:33431 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:32:47,759 DEBUG [RS:0;fadb2fe0ba32:33431 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/fadb2fe0ba32:0, corePoolSize=1, maxPoolSize=1
2024-11-24T04:32:47,759 DEBUG [RS:0;fadb2fe0ba32:33431 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/fadb2fe0ba32:0, corePoolSize=3, maxPoolSize=3
2024-11-24T04:32:47,759 DEBUG [RS:0;fadb2fe0ba32:33431 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/fadb2fe0ba32:0, corePoolSize=3, maxPoolSize=3
2024-11-24T04:32:47,759 INFO  [RS:0;fadb2fe0ba32:33431 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-24T04:32:47,759 INFO  [RS:0;fadb2fe0ba32:33431 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-24T04:32:47,759 INFO  [RS:0;fadb2fe0ba32:33431 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-24T04:32:47,759 INFO  [RS:0;fadb2fe0ba32:33431 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-24T04:32:47,759 INFO  [RS:0;fadb2fe0ba32:33431 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-24T04:32:47,759 INFO  [RS:0;fadb2fe0ba32:33431 {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,33431,1732422767512-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-24T04:32:47,774 INFO  [RS:0;fadb2fe0ba32:33431 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-24T04:32:47,774 INFO  [RS:0;fadb2fe0ba32:33431 {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,33431,1732422767512-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-24T04:32:47,774 INFO  [RS:0;fadb2fe0ba32:33431 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T04:32:47,774 INFO  [RS:0;fadb2fe0ba32:33431 {}] regionserver.Replication(171): fadb2fe0ba32,33431,1732422767512 started
2024-11-24T04:32:47,787 INFO  [RS:0;fadb2fe0ba32:33431 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T04:32:47,788 INFO  [RS:0;fadb2fe0ba32:33431 {}] regionserver.HRegionServer(1482): Serving as fadb2fe0ba32,33431,1732422767512, RpcServer on fadb2fe0ba32/172.17.0.2:33431, sessionid=0x101483f7df00001
2024-11-24T04:32:47,788 DEBUG [RS:0;fadb2fe0ba32:33431 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-24T04:32:47,788 DEBUG [RS:0;fadb2fe0ba32:33431 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager fadb2fe0ba32,33431,1732422767512
2024-11-24T04:32:47,788 DEBUG [RS:0;fadb2fe0ba32:33431 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fadb2fe0ba32,33431,1732422767512'
2024-11-24T04:32:47,788 DEBUG [RS:0;fadb2fe0ba32:33431 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-24T04:32:47,788 DEBUG [RS:0;fadb2fe0ba32:33431 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-24T04:32:47,789 DEBUG [RS:0;fadb2fe0ba32:33431 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-24T04:32:47,789 DEBUG [RS:0;fadb2fe0ba32:33431 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-24T04:32:47,789 DEBUG [RS:0;fadb2fe0ba32:33431 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager fadb2fe0ba32,33431,1732422767512
2024-11-24T04:32:47,789 DEBUG [RS:0;fadb2fe0ba32:33431 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fadb2fe0ba32,33431,1732422767512'
2024-11-24T04:32:47,789 DEBUG [RS:0;fadb2fe0ba32:33431 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-24T04:32:47,789 DEBUG [RS:0;fadb2fe0ba32:33431 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-24T04:32:47,789 DEBUG [RS:0;fadb2fe0ba32:33431 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-24T04:32:47,789 INFO  [RS:0;fadb2fe0ba32:33431 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-24T04:32:47,789 INFO  [RS:0;fadb2fe0ba32:33431 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-24T04:32:47,837 WARN  [fadb2fe0ba32:45215 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-11-24T04:32:47,874 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:47,874 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:47,891 INFO  [RS:0;fadb2fe0ba32:33431 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fadb2fe0ba32%2C33431%2C1732422767512, suffix=, logDir=hdfs://localhost:37089/user/jenkins/test-data/6646e03d-3e80-85d7-fac8-e27673f4f898/WALs/fadb2fe0ba32,33431,1732422767512, archiveDir=hdfs://localhost:37089/user/jenkins/test-data/6646e03d-3e80-85d7-fac8-e27673f4f898/oldWALs, maxLogs=32
2024-11-24T04:32:47,891 INFO  [RS:0;fadb2fe0ba32:33431 {}] monitor.StreamSlowMonitor(122): New stream slow monitor fadb2fe0ba32%2C33431%2C1732422767512.1732422767891
2024-11-24T04:32:47,896 INFO  [RS:0;fadb2fe0ba32:33431 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6646e03d-3e80-85d7-fac8-e27673f4f898/WALs/fadb2fe0ba32,33431,1732422767512/fadb2fe0ba32%2C33431%2C1732422767512.1732422767891
2024-11-24T04:32:47,897 DEBUG [RS:0;fadb2fe0ba32:33431 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43777:43777),(127.0.0.1/127.0.0.1:45377:45377)]
2024-11-24T04:32:48,088 DEBUG [fadb2fe0ba32:45215 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-24T04:32:48,088 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=fadb2fe0ba32,33431,1732422767512
2024-11-24T04:32:48,089 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fadb2fe0ba32,33431,1732422767512, state=OPENING
2024-11-24T04:32:48,091 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-24T04:32:48,094 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45215-0x101483f7df00000, quorum=127.0.0.1:56125, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:32:48,094 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33431-0x101483f7df00001, quorum=127.0.0.1:56125, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:32:48,094 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-24T04:32:48,094 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-24T04:32:48,094 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-24T04:32:48,094 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=fadb2fe0ba32,33431,1732422767512}]
2024-11-24T04:32:48,246 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-24T04:32:48,248 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-17-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44129, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-24T04:32:48,251 INFO  [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-24T04:32:48,252 INFO  [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-24T04:32:48,253 INFO  [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fadb2fe0ba32%2C33431%2C1732422767512.meta, suffix=.meta, logDir=hdfs://localhost:37089/user/jenkins/test-data/6646e03d-3e80-85d7-fac8-e27673f4f898/WALs/fadb2fe0ba32,33431,1732422767512, archiveDir=hdfs://localhost:37089/user/jenkins/test-data/6646e03d-3e80-85d7-fac8-e27673f4f898/oldWALs, maxLogs=32
2024-11-24T04:32:48,253 INFO  [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor fadb2fe0ba32%2C33431%2C1732422767512.meta.1732422768253.meta
2024-11-24T04:32:48,258 INFO  [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6646e03d-3e80-85d7-fac8-e27673f4f898/WALs/fadb2fe0ba32,33431,1732422767512/fadb2fe0ba32%2C33431%2C1732422767512.meta.1732422768253.meta
2024-11-24T04:32:48,262 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43777:43777),(127.0.0.1/127.0.0.1:45377:45377)]
2024-11-24T04:32:48,268 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-24T04:32:48,269 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-24T04:32:48,269 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-24T04:32:48,269 INFO  [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-24T04:32:48,269 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-24T04:32:48,269 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-24T04:32:48,269 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-24T04:32:48,269 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-24T04:32:48,270 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-24T04:32:48,271 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-24T04:32:48,271 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:32:48,271 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T04:32:48,271 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-24T04:32:48,272 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-24T04:32:48,272 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:32:48,272 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T04:32:48,272 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-24T04:32:48,273 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-24T04:32:48,273 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:32:48,273 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T04:32:48,273 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-24T04:32:48,274 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-24T04:32:48,274 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-24T04:32:48,274 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-24T04:32:48,274 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-24T04:32:48,275 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37089/user/jenkins/test-data/6646e03d-3e80-85d7-fac8-e27673f4f898/data/hbase/meta/1588230740
2024-11-24T04:32:48,276 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37089/user/jenkins/test-data/6646e03d-3e80-85d7-fac8-e27673f4f898/data/hbase/meta/1588230740
2024-11-24T04:32:48,277 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-24T04:32:48,277 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-24T04:32:48,277 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-24T04:32:48,278 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-24T04:32:48,279 INFO  [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=731120, jitterRate=-0.0703335702419281}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-24T04:32:48,279 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-24T04:32:48,279 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1732422768269Writing region info on filesystem at 1732422768269Initializing all the Stores at 1732422768270 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732422768270Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732422768270Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732422768270Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732422768270Cleaning up temporary data from old regions at 1732422768277 (+7 ms)Running coprocessor post-open hooks at 1732422768279 (+2 ms)Region opened successfully at 1732422768279
2024-11-24T04:32:48,280 INFO  [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732422768246
2024-11-24T04:32:48,282 DEBUG [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-24T04:32:48,282 INFO  [RS_OPEN_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-24T04:32:48,283 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=fadb2fe0ba32,33431,1732422767512
2024-11-24T04:32:48,284 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fadb2fe0ba32,33431,1732422767512, state=OPEN
2024-11-24T04:32:48,288 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33431-0x101483f7df00001, quorum=127.0.0.1:56125, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-24T04:32:48,288 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45215-0x101483f7df00000, quorum=127.0.0.1:56125, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-24T04:32:48,288 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=fadb2fe0ba32,33431,1732422767512
2024-11-24T04:32:48,288 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-24T04:32:48,288 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-24T04:32:48,290 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-24T04:32:48,290 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fadb2fe0ba32,33431,1732422767512 in 194 msec
2024-11-24T04:32:48,292 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-24T04:32:48,292 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 605 msec
2024-11-24T04:32:48,292 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-24T04:32:48,292 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-24T04:32:48,293 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-24T04:32:48,293 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fadb2fe0ba32,33431,1732422767512, seqNum=-1]
2024-11-24T04:32:48,294 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-24T04:32:48,295 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-17-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50499, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-24T04:32:48,299 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 642 msec
2024-11-24T04:32:48,299 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732422768299, completionTime=-1
2024-11-24T04:32:48,299 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-24T04:32:48,299 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-24T04:32:48,300 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-24T04:32:48,300 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732422828300
2024-11-24T04:32:48,301 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732422888300
2024-11-24T04:32:48,301 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 1 msec
2024-11-24T04:32:48,301 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,45215,1732422767469-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-24T04:32:48,301 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,45215,1732422767469-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T04:32:48,301 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,45215,1732422767469-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T04:32:48,301 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-fadb2fe0ba32:45215, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T04:32:48,301 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-24T04:32:48,301 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-24T04:32:48,302 DEBUG [master/fadb2fe0ba32:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-24T04:32:48,305 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.759sec
2024-11-24T04:32:48,305 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-24T04:32:48,305 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-24T04:32:48,305 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-24T04:32:48,305 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-24T04:32:48,305 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-24T04:32:48,305 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,45215,1732422767469-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-24T04:32:48,305 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,45215,1732422767469-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-24T04:32:48,307 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-24T04:32:48,307 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-24T04:32:48,307 INFO  [master/fadb2fe0ba32:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fadb2fe0ba32,45215,1732422767469-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-24T04:32:48,326 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@392cf3b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-24T04:32:48,326 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fadb2fe0ba32,45215,-1 for getting cluster id
2024-11-24T04:32:48,326 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-24T04:32:48,328 DEBUG [HMaster-EventLoopGroup-16-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fe5b4538-2f36-43f4-993b-f270d4709ca1'
2024-11-24T04:32:48,328 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-24T04:32:48,328 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fe5b4538-2f36-43f4-993b-f270d4709ca1"

2024-11-24T04:32:48,328 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63bb83a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-24T04:32:48,328 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fadb2fe0ba32,45215,-1]
2024-11-24T04:32:48,329 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-24T04:32:48,329 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T04:32:48,330 INFO  [HMaster-EventLoopGroup-16-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36790, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-24T04:32:48,330 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1557ab96, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-24T04:32:48,331 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-24T04:32:48,331 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fadb2fe0ba32,33431,1732422767512, seqNum=-1]
2024-11-24T04:32:48,332 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-24T04:32:48,333 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-17-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39572, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-24T04:32:48,334 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=fadb2fe0ba32,45215,1732422767469
2024-11-24T04:32:48,334 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-24T04:32:48,337 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-24T04:32:48,337 INFO  [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-24T04:32:48,338 INFO  [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=test.com%2C8080%2C1, suffix=, logDir=hdfs://localhost:37089/user/jenkins/test-data/6646e03d-3e80-85d7-fac8-e27673f4f898/WALs/test.com,8080,1, archiveDir=hdfs://localhost:37089/user/jenkins/test-data/6646e03d-3e80-85d7-fac8-e27673f4f898/oldWALs, maxLogs=32
2024-11-24T04:32:48,339 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1732422768339
2024-11-24T04:32:48,343 INFO  [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6646e03d-3e80-85d7-fac8-e27673f4f898/WALs/test.com,8080,1/test.com%2C8080%2C1.1732422768339
2024-11-24T04:32:48,344 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45377:45377),(127.0.0.1/127.0.0.1:43777:43777)]
2024-11-24T04:32:48,345 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1732422768345
2024-11-24T04:32:48,349 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:32:48,349 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:32:48,349 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:32:48,349 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:32:48,349 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:32:48,349 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/6646e03d-3e80-85d7-fac8-e27673f4f898/WALs/test.com,8080,1/test.com%2C8080%2C1.1732422768339 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/6646e03d-3e80-85d7-fac8-e27673f4f898/WALs/test.com,8080,1/test.com%2C8080%2C1.1732422768345
2024-11-24T04:32:48,350 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45377:45377),(127.0.0.1/127.0.0.1:43777:43777)]
2024-11-24T04:32:48,350 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:37089/user/jenkins/test-data/6646e03d-3e80-85d7-fac8-e27673f4f898/WALs/test.com,8080,1/test.com%2C8080%2C1.1732422768339 is not closed yet, will try archiving it next time
2024-11-24T04:32:48,350 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:32:48,350 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:32:48,350 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:32:48,350 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:32:48,351 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38705 is added to blk_1073741835_1011 (size=93)
2024-11-24T04:32:48,351 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:32:48,351 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40251 is added to blk_1073741835_1011 (size=93)
2024-11-24T04:32:48,353 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38705 is added to blk_1073741836_1012 (size=93)
2024-11-24T04:32:48,353 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40251 is added to blk_1073741836_1012 (size=93)
2024-11-24T04:32:48,752 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:37089/user/jenkins/test-data/6646e03d-3e80-85d7-fac8-e27673f4f898/WALs/test.com,8080,1/test.com%2C8080%2C1.1732422768339 to hdfs://localhost:37089/user/jenkins/test-data/6646e03d-3e80-85d7-fac8-e27673f4f898/oldWALs/test.com%2C8080%2C1.1732422768339
2024-11-24T04:32:48,754 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/6646e03d-3e80-85d7-fac8-e27673f4f898/oldWALs
2024-11-24T04:32:48,754 INFO  [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog test.com%2C8080%2C1:(num 1732422768345)
2024-11-24T04:32:48,754 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-24T04:32:48,755 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-24T04:32:48,755 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-24T04:32:48,755 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T04:32:48,755 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T04:32:48,755 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-24T04:32:48,755 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-24T04:32:48,755 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1247586563, stopped=false
2024-11-24T04:32:48,755 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=fadb2fe0ba32,45215,1732422767469
2024-11-24T04:32:48,757 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33431-0x101483f7df00001, quorum=127.0.0.1:56125, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-24T04:32:48,757 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45215-0x101483f7df00000, quorum=127.0.0.1:56125, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-24T04:32:48,757 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33431-0x101483f7df00001, quorum=127.0.0.1:56125, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:32:48,757 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45215-0x101483f7df00000, quorum=127.0.0.1:56125, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:32:48,757 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-24T04:32:48,757 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-24T04:32:48,757 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-24T04:32:48,757 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T04:32:48,757 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'fadb2fe0ba32,33431,1732422767512' *****
2024-11-24T04:32:48,757 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-24T04:32:48,758 INFO  [RS:0;fadb2fe0ba32:33431 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-24T04:32:48,758 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-24T04:32:48,758 INFO  [RS:0;fadb2fe0ba32:33431 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-24T04:32:48,758 INFO  [RS:0;fadb2fe0ba32:33431 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-24T04:32:48,758 INFO  [RS:0;fadb2fe0ba32:33431 {}] regionserver.HRegionServer(959): stopping server fadb2fe0ba32,33431,1732422767512
2024-11-24T04:32:48,758 INFO  [RS:0;fadb2fe0ba32:33431 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-24T04:32:48,758 INFO  [RS:0;fadb2fe0ba32:33431 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;fadb2fe0ba32:33431.
2024-11-24T04:32:48,758 DEBUG [RS:0;fadb2fe0ba32:33431 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-24T04:32:48,758 DEBUG [RS:0;fadb2fe0ba32:33431 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T04:32:48,758 INFO  [RS:0;fadb2fe0ba32:33431 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-24T04:32:48,758 INFO  [RS:0;fadb2fe0ba32:33431 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-24T04:32:48,758 INFO  [RS:0;fadb2fe0ba32:33431 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-24T04:32:48,758 INFO  [RS:0;fadb2fe0ba32:33431 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-24T04:32:48,759 INFO  [RS:0;fadb2fe0ba32:33431 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close
2024-11-24T04:32:48,759 DEBUG [RS:0;fadb2fe0ba32:33431 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740}
2024-11-24T04:32:48,759 DEBUG [RS:0;fadb2fe0ba32:33431 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-24T04:32:48,759 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-24T04:32:48,759 INFO  [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-24T04:32:48,759 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-24T04:32:48,759 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-24T04:32:48,759 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-24T04:32:48,759 INFO  [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=74 B heapSize=1.22 KB
2024-11-24T04:32:48,761 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45215-0x101483f7df00000, quorum=127.0.0.1:56125, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-24T04:32:48,761 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33431-0x101483f7df00001, quorum=127.0.0.1:56125, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-24T04:32:48,771 INFO  [regionserver/fadb2fe0ba32:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-11-24T04:32:48,772 INFO  [regionserver/fadb2fe0ba32:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-11-24T04:32:48,775 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37089/user/jenkins/test-data/6646e03d-3e80-85d7-fac8-e27673f4f898/data/hbase/meta/1588230740/.tmp/ns/77be6a4b27a348f5887bade78cc17c34 is 43, key is default/ns:d/1732422768295/Put/seqid=0
2024-11-24T04:32:48,779 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40251 is added to blk_1073741837_1013 (size=5153)
2024-11-24T04:32:48,780 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38705 is added to blk_1073741837_1013 (size=5153)
2024-11-24T04:32:48,780 INFO  [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:37089/user/jenkins/test-data/6646e03d-3e80-85d7-fac8-e27673f4f898/data/hbase/meta/1588230740/.tmp/ns/77be6a4b27a348f5887bade78cc17c34
2024-11-24T04:32:48,785 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37089/user/jenkins/test-data/6646e03d-3e80-85d7-fac8-e27673f4f898/data/hbase/meta/1588230740/.tmp/ns/77be6a4b27a348f5887bade78cc17c34 as hdfs://localhost:37089/user/jenkins/test-data/6646e03d-3e80-85d7-fac8-e27673f4f898/data/hbase/meta/1588230740/ns/77be6a4b27a348f5887bade78cc17c34
2024-11-24T04:32:48,789 INFO  [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37089/user/jenkins/test-data/6646e03d-3e80-85d7-fac8-e27673f4f898/data/hbase/meta/1588230740/ns/77be6a4b27a348f5887bade78cc17c34, entries=2, sequenceid=6, filesize=5.0 K
2024-11-24T04:32:48,790 INFO  [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 31ms, sequenceid=6, compaction requested=false
2024-11-24T04:32:48,793 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37089/user/jenkins/test-data/6646e03d-3e80-85d7-fac8-e27673f4f898/data/hbase/meta/1588230740/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1
2024-11-24T04:32:48,794 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-24T04:32:48,794 INFO  [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-24T04:32:48,794 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1732422768759Running coprocessor pre-close hooks at 1732422768759Disabling compacts and flushes for region at 1732422768759Disabling writes for close at 1732422768759Obtaining lock to block concurrent updates at 1732422768759Preparing flush snapshotting stores in 1588230740 at 1732422768759Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=74, getHeapSize=1184, getOffHeapSize=0, getCellsCount=2 at 1732422768759Flushing stores of hbase:meta,,1.1588230740 at 1732422768760 (+1 ms)Flushing 1588230740/ns: creating writer at 1732422768760Flushing 1588230740/ns: appending metadata at 1732422768775 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1732422768775Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7d5e814f: reopening flushed file at 1732422768784 (+9 ms)Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 31ms, sequenceid=6, compaction requested=false at 1732422768790 (+6 ms)Writing region close event to WAL at 1732422768790Running coprocessor post-close hooks at 1732422768793 (+3 ms)Closed at 1732422768794 (+1 ms)
2024-11-24T04:32:48,794 DEBUG [RS_CLOSE_META-regionserver/fadb2fe0ba32:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-24T04:32:48,875 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,40681,1732422576481/fadb2fe0ba32%2C40681%2C1732422576481.meta.1732422577361.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:48,875 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40187/user/jenkins/test-data/672d3a1b-beb3-474a-a572-0b071dfec2bc/WALs/fadb2fe0ba32,44377,1732422577562/fadb2fe0ba32%2C44377%2C1732422577562.1732422577749
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-24T04:32:48,959 INFO  [RS:0;fadb2fe0ba32:33431 {}] regionserver.HRegionServer(976): stopping server fadb2fe0ba32,33431,1732422767512; all regions closed.
2024-11-24T04:32:48,959 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:32:48,959 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:32:48,959 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:32:48,960 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:32:48,960 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:32:48,961 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40251 is added to blk_1073741834_1010 (size=1152)
2024-11-24T04:32:48,962 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38705 is added to blk_1073741834_1010 (size=1152)
2024-11-24T04:32:48,964 DEBUG [RS:0;fadb2fe0ba32:33431 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/6646e03d-3e80-85d7-fac8-e27673f4f898/oldWALs
2024-11-24T04:32:48,964 INFO  [RS:0;fadb2fe0ba32:33431 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog fadb2fe0ba32%2C33431%2C1732422767512.meta:.meta(num 1732422768253)
2024-11-24T04:32:48,964 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:32:48,964 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:32:48,964 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:32:48,964 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:32:48,964 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:32:48,966 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40251 is added to blk_1073741833_1009 (size=93)
2024-11-24T04:32:48,966 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38705 is added to blk_1073741833_1009 (size=93)
2024-11-24T04:32:48,968 DEBUG [RS:0;fadb2fe0ba32:33431 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/6646e03d-3e80-85d7-fac8-e27673f4f898/oldWALs
2024-11-24T04:32:48,968 INFO  [RS:0;fadb2fe0ba32:33431 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog fadb2fe0ba32%2C33431%2C1732422767512:(num 1732422767891)
2024-11-24T04:32:48,968 DEBUG [RS:0;fadb2fe0ba32:33431 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-24T04:32:48,968 INFO  [RS:0;fadb2fe0ba32:33431 {}] regionserver.LeaseManager(133): Closed leases
2024-11-24T04:32:48,968 INFO  [RS:0;fadb2fe0ba32:33431 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-24T04:32:48,968 INFO  [RS:0;fadb2fe0ba32:33431 {}] hbase.ChoreService(370): Chore service for: regionserver/fadb2fe0ba32:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-24T04:32:48,969 INFO  [RS:0;fadb2fe0ba32:33431 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-24T04:32:48,969 INFO  [regionserver/fadb2fe0ba32:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-24T04:32:48,969 INFO  [RS:0;fadb2fe0ba32:33431 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33431
2024-11-24T04:32:48,972 INFO  [RS:0;fadb2fe0ba32:33431 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-24T04:32:48,972 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45215-0x101483f7df00000, quorum=127.0.0.1:56125, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-24T04:32:48,972 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33431-0x101483f7df00001, quorum=127.0.0.1:56125, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/fadb2fe0ba32,33431,1732422767512
2024-11-24T04:32:48,973 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [fadb2fe0ba32,33431,1732422767512]
2024-11-24T04:32:48,974 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/fadb2fe0ba32,33431,1732422767512 already deleted, retry=false
2024-11-24T04:32:48,974 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; fadb2fe0ba32,33431,1732422767512 expired; onlineServers=0
2024-11-24T04:32:48,974 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'fadb2fe0ba32,45215,1732422767469' *****
2024-11-24T04:32:48,974 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-24T04:32:48,974 INFO  [M:0;fadb2fe0ba32:45215 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-24T04:32:48,974 INFO  [M:0;fadb2fe0ba32:45215 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-24T04:32:48,974 DEBUG [M:0;fadb2fe0ba32:45215 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-24T04:32:48,974 DEBUG [M:0;fadb2fe0ba32:45215 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-24T04:32:48,974 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-24T04:32:48,974 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster-HFileCleaner.large.0-1732422767660 {}] cleaner.HFileCleaner(306): Exit Thread[master/fadb2fe0ba32:0:becomeActiveMaster-HFileCleaner.large.0-1732422767660,5,FailOnTimeoutGroup]
2024-11-24T04:32:48,974 DEBUG [master/fadb2fe0ba32:0:becomeActiveMaster-HFileCleaner.small.0-1732422767661 {}] cleaner.HFileCleaner(306): Exit Thread[master/fadb2fe0ba32:0:becomeActiveMaster-HFileCleaner.small.0-1732422767661,5,FailOnTimeoutGroup]
2024-11-24T04:32:48,975 INFO  [M:0;fadb2fe0ba32:45215 {}] hbase.ChoreService(370): Chore service for: master/fadb2fe0ba32:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-24T04:32:48,975 INFO  [M:0;fadb2fe0ba32:45215 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-24T04:32:48,975 DEBUG [M:0;fadb2fe0ba32:45215 {}] master.HMaster(1795): Stopping service threads
2024-11-24T04:32:48,975 INFO  [M:0;fadb2fe0ba32:45215 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-24T04:32:48,975 INFO  [M:0;fadb2fe0ba32:45215 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-24T04:32:48,975 INFO  [M:0;fadb2fe0ba32:45215 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-24T04:32:48,975 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-24T04:32:48,976 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45215-0x101483f7df00000, quorum=127.0.0.1:56125, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-24T04:32:48,976 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45215-0x101483f7df00000, quorum=127.0.0.1:56125, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-24T04:32:48,976 DEBUG [M:0;fadb2fe0ba32:45215 {}] zookeeper.ZKUtil(347): master:45215-0x101483f7df00000, quorum=127.0.0.1:56125, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-24T04:32:48,976 WARN  [M:0;fadb2fe0ba32:45215 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-24T04:32:48,976 INFO  [M:0;fadb2fe0ba32:45215 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:37089/user/jenkins/test-data/6646e03d-3e80-85d7-fac8-e27673f4f898/.lastflushedseqids
2024-11-24T04:32:48,981 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38705 is added to blk_1073741838_1014 (size=99)
2024-11-24T04:32:48,981 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40251 is added to blk_1073741838_1014 (size=99)
2024-11-24T04:32:48,981 INFO  [M:0;fadb2fe0ba32:45215 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-24T04:32:48,982 INFO  [M:0;fadb2fe0ba32:45215 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-24T04:32:48,982 DEBUG [M:0;fadb2fe0ba32:45215 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-24T04:32:48,982 INFO  [M:0;fadb2fe0ba32:45215 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T04:32:48,982 DEBUG [M:0;fadb2fe0ba32:45215 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T04:32:48,982 DEBUG [M:0;fadb2fe0ba32:45215 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-24T04:32:48,982 DEBUG [M:0;fadb2fe0ba32:45215 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T04:32:48,982 INFO  [M:0;fadb2fe0ba32:45215 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.67 KB heapSize=11.34 KB
2024-11-24T04:32:48,997 DEBUG [M:0;fadb2fe0ba32:45215 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37089/user/jenkins/test-data/6646e03d-3e80-85d7-fac8-e27673f4f898/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1a3a5b8674d0411e90d677faabdcb528 is 82, key is hbase:meta,,1/info:regioninfo/1732422768283/Put/seqid=0
2024-11-24T04:32:49,001 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40251 is added to blk_1073741839_1015 (size=5672)
2024-11-24T04:32:49,001 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38705 is added to blk_1073741839_1015 (size=5672)
2024-11-24T04:32:49,002 INFO  [M:0;fadb2fe0ba32:45215 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:37089/user/jenkins/test-data/6646e03d-3e80-85d7-fac8-e27673f4f898/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1a3a5b8674d0411e90d677faabdcb528
2024-11-24T04:32:49,019 DEBUG [M:0;fadb2fe0ba32:45215 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37089/user/jenkins/test-data/6646e03d-3e80-85d7-fac8-e27673f4f898/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0c3153dab3fa4b59922adeeb08cd1b79 is 240, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1732422768298/Put/seqid=0
2024-11-24T04:32:49,024 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40251 is added to blk_1073741840_1016 (size=5275)
2024-11-24T04:32:49,024 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38705 is added to blk_1073741840_1016 (size=5275)
2024-11-24T04:32:49,024 INFO  [M:0;fadb2fe0ba32:45215 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:37089/user/jenkins/test-data/6646e03d-3e80-85d7-fac8-e27673f4f898/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0c3153dab3fa4b59922adeeb08cd1b79
2024-11-24T04:32:49,043 DEBUG [M:0;fadb2fe0ba32:45215 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37089/user/jenkins/test-data/6646e03d-3e80-85d7-fac8-e27673f4f898/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/768652acb9a948bf91eff1e8c5010def is 69, key is fadb2fe0ba32,33431,1732422767512/rs:state/1732422767747/Put/seqid=0
2024-11-24T04:32:49,046 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40251 is added to blk_1073741841_1017 (size=5156)
2024-11-24T04:32:49,047 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38705 is added to blk_1073741841_1017 (size=5156)
2024-11-24T04:32:49,047 INFO  [M:0;fadb2fe0ba32:45215 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:37089/user/jenkins/test-data/6646e03d-3e80-85d7-fac8-e27673f4f898/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/768652acb9a948bf91eff1e8c5010def
2024-11-24T04:32:49,065 DEBUG [M:0;fadb2fe0ba32:45215 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37089/user/jenkins/test-data/6646e03d-3e80-85d7-fac8-e27673f4f898/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/594200bfa25e47c3a7cf6a1f3a66a407 is 52, key is load_balancer_on/state:d/1732422768336/Put/seqid=0
2024-11-24T04:32:49,068 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40251 is added to blk_1073741842_1018 (size=5056)
2024-11-24T04:32:49,068 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38705 is added to blk_1073741842_1018 (size=5056)
2024-11-24T04:32:49,069 INFO  [M:0;fadb2fe0ba32:45215 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:37089/user/jenkins/test-data/6646e03d-3e80-85d7-fac8-e27673f4f898/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/594200bfa25e47c3a7cf6a1f3a66a407
2024-11-24T04:32:49,073 DEBUG [M:0;fadb2fe0ba32:45215 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37089/user/jenkins/test-data/6646e03d-3e80-85d7-fac8-e27673f4f898/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1a3a5b8674d0411e90d677faabdcb528 as hdfs://localhost:37089/user/jenkins/test-data/6646e03d-3e80-85d7-fac8-e27673f4f898/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1a3a5b8674d0411e90d677faabdcb528
2024-11-24T04:32:49,073 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33431-0x101483f7df00001, quorum=127.0.0.1:56125, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-24T04:32:49,073 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33431-0x101483f7df00001, quorum=127.0.0.1:56125, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-24T04:32:49,073 INFO  [RS:0;fadb2fe0ba32:33431 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-24T04:32:49,073 INFO  [RS:0;fadb2fe0ba32:33431 {}] regionserver.HRegionServer(1031): Exiting; stopping=fadb2fe0ba32,33431,1732422767512; zookeeper connection closed.
2024-11-24T04:32:49,073 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5f3ca3a9 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5f3ca3a9
2024-11-24T04:32:49,074 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-24T04:32:49,077 INFO  [M:0;fadb2fe0ba32:45215 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37089/user/jenkins/test-data/6646e03d-3e80-85d7-fac8-e27673f4f898/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1a3a5b8674d0411e90d677faabdcb528, entries=8, sequenceid=29, filesize=5.5 K
2024-11-24T04:32:49,077 DEBUG [M:0;fadb2fe0ba32:45215 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37089/user/jenkins/test-data/6646e03d-3e80-85d7-fac8-e27673f4f898/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0c3153dab3fa4b59922adeeb08cd1b79 as hdfs://localhost:37089/user/jenkins/test-data/6646e03d-3e80-85d7-fac8-e27673f4f898/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0c3153dab3fa4b59922adeeb08cd1b79
2024-11-24T04:32:49,082 INFO  [M:0;fadb2fe0ba32:45215 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37089/user/jenkins/test-data/6646e03d-3e80-85d7-fac8-e27673f4f898/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0c3153dab3fa4b59922adeeb08cd1b79, entries=3, sequenceid=29, filesize=5.2 K
2024-11-24T04:32:49,083 DEBUG [M:0;fadb2fe0ba32:45215 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37089/user/jenkins/test-data/6646e03d-3e80-85d7-fac8-e27673f4f898/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/768652acb9a948bf91eff1e8c5010def as hdfs://localhost:37089/user/jenkins/test-data/6646e03d-3e80-85d7-fac8-e27673f4f898/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/768652acb9a948bf91eff1e8c5010def
2024-11-24T04:32:49,087 INFO  [M:0;fadb2fe0ba32:45215 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37089/user/jenkins/test-data/6646e03d-3e80-85d7-fac8-e27673f4f898/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/768652acb9a948bf91eff1e8c5010def, entries=1, sequenceid=29, filesize=5.0 K
2024-11-24T04:32:49,088 DEBUG [M:0;fadb2fe0ba32:45215 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37089/user/jenkins/test-data/6646e03d-3e80-85d7-fac8-e27673f4f898/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/594200bfa25e47c3a7cf6a1f3a66a407 as hdfs://localhost:37089/user/jenkins/test-data/6646e03d-3e80-85d7-fac8-e27673f4f898/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/594200bfa25e47c3a7cf6a1f3a66a407
2024-11-24T04:32:49,092 INFO  [M:0;fadb2fe0ba32:45215 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37089/user/jenkins/test-data/6646e03d-3e80-85d7-fac8-e27673f4f898/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/594200bfa25e47c3a7cf6a1f3a66a407, entries=1, sequenceid=29, filesize=4.9 K
2024-11-24T04:32:49,094 INFO  [M:0;fadb2fe0ba32:45215 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 111ms, sequenceid=29, compaction requested=false
2024-11-24T04:32:49,095 INFO  [M:0;fadb2fe0ba32:45215 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-24T04:32:49,095 DEBUG [M:0;fadb2fe0ba32:45215 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1732422768982Disabling compacts and flushes for region at 1732422768982Disabling writes for close at 1732422768982Obtaining lock to block concurrent updates at 1732422768982Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732422768982Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7850, getHeapSize=11544, getOffHeapSize=0, getCellsCount=36 at 1732422768982Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732422768983 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732422768983Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732422768997 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732422768997Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732422769005 (+8 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732422769019 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732422769019Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732422769028 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732422769042 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732422769042Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732422769051 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732422769064 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732422769064Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1ca50a79: reopening flushed file at 1732422769072 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@39671733: reopening flushed file at 1732422769077 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@326e9088: reopening flushed file at 1732422769082 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4389433e: reopening flushed file at 1732422769087 (+5 ms)Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 111ms, sequenceid=29, compaction requested=false at 1732422769094 (+7 ms)Writing region close event to WAL at 1732422769095 (+1 ms)Closed at 1732422769095
2024-11-24T04:32:49,096 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:32:49,096 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:32:49,096 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:32:49,096 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:32:49,096 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-24T04:32:49,098 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38705 is added to blk_1073741830_1006 (size=10311)
2024-11-24T04:32:49,098 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40251 is added to blk_1073741830_1006 (size=10311)
2024-11-24T04:32:49,099 INFO  [M:0;fadb2fe0ba32:45215 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-24T04:32:49,099 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-24T04:32:49,099 INFO  [M:0;fadb2fe0ba32:45215 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45215
2024-11-24T04:32:49,099 INFO  [M:0;fadb2fe0ba32:45215 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-24T04:32:49,207 INFO  [M:0;fadb2fe0ba32:45215 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-24T04:32:49,207 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45215-0x101483f7df00000, quorum=127.0.0.1:56125, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-24T04:32:49,207 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45215-0x101483f7df00000, quorum=127.0.0.1:56125, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-24T04:32:49,209 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3698cd86{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T04:32:49,210 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@418bce8c{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-24T04:32:49,210 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-24T04:32:49,210 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@35a6030c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-24T04:32:49,210 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@677f535e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/239376e9-b709-06b1-2009-9af67614e40c/hadoop.log.dir/,STOPPED}
2024-11-24T04:32:49,212 WARN  [BP-1321689454-172.17.0.2-1732422766805 heartbeating to localhost/127.0.0.1:37089 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-24T04:32:49,212 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-24T04:32:49,212 WARN  [BP-1321689454-172.17.0.2-1732422766805 heartbeating to localhost/127.0.0.1:37089 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1321689454-172.17.0.2-1732422766805 (Datanode Uuid 8683ebfa-83a3-4661-af86-01fabb3439d6) service to localhost/127.0.0.1:37089
2024-11-24T04:32:49,212 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-24T04:32:49,212 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/239376e9-b709-06b1-2009-9af67614e40c/cluster_ba9c276b-166c-9d23-ed5c-f05f16006543/data/data3/current/BP-1321689454-172.17.0.2-1732422766805 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T04:32:49,213 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/239376e9-b709-06b1-2009-9af67614e40c/cluster_ba9c276b-166c-9d23-ed5c-f05f16006543/data/data4/current/BP-1321689454-172.17.0.2-1732422766805 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T04:32:49,213 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-24T04:32:49,215 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@d6e133d{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-24T04:32:49,215 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@652ac650{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-24T04:32:49,215 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-24T04:32:49,215 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1802ed83{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-24T04:32:49,215 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@142dd349{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/239376e9-b709-06b1-2009-9af67614e40c/hadoop.log.dir/,STOPPED}
2024-11-24T04:32:49,216 WARN  [BP-1321689454-172.17.0.2-1732422766805 heartbeating to localhost/127.0.0.1:37089 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-24T04:32:49,216 WARN  [BP-1321689454-172.17.0.2-1732422766805 heartbeating to localhost/127.0.0.1:37089 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1321689454-172.17.0.2-1732422766805 (Datanode Uuid 9d57edc0-9bb2-4f98-bae5-0b3df1e6dd01) service to localhost/127.0.0.1:37089
2024-11-24T04:32:49,217 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/239376e9-b709-06b1-2009-9af67614e40c/cluster_ba9c276b-166c-9d23-ed5c-f05f16006543/data/data1/current/BP-1321689454-172.17.0.2-1732422766805 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T04:32:49,217 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/239376e9-b709-06b1-2009-9af67614e40c/cluster_ba9c276b-166c-9d23-ed5c-f05f16006543/data/data2/current/BP-1321689454-172.17.0.2-1732422766805 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-24T04:32:49,217 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-24T04:32:49,217 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-24T04:32:49,217 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-24T04:32:49,223 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2a7c9e07{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-24T04:32:49,224 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@225265b7{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-24T04:32:49,224 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-24T04:32:49,224 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@64825638{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-24T04:32:49,224 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@63ebf311{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/239376e9-b709-06b1-2009-9af67614e40c/hadoop.log.dir/,STOPPED}
2024-11-24T04:32:49,230 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-24T04:32:49,248 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-24T04:32:49,258 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=267 (was 230)
Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:37089 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.7@localhost:37089
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-44-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-42-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-45-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-16-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-16-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-43-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37089
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-16-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:37089
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-42-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-44-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:37089 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37089
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-45-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-42-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: globalEventExecutor-1-21
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	app//io.netty.util.concurrent.GlobalEventExecutor.takeTask(GlobalEventExecutor.java:113)
	app//io.netty.util.concurrent.GlobalEventExecutor$TaskRunner.run(GlobalEventExecutor.java:259)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-43-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-44-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37089
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-43-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-45-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:37089 from jenkins.hfs.7
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)
 - Thread LEAK? -, OpenFileDescriptor=534 (was 503) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=22 (was 16) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6395 (was 6403)